From cf8987b843c56a57b8cfa4f9119f3d35486a698e Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 20:18:18 -0400 Subject: [PATCH 01/14] docs(evals): plan capped live runner --- DOCUMENTATION_INDEX.md | 5 +- ...-07-18-h1-live-evaluation-runner-design.md | 194 ++++++ .../2026-07-18-h1-live-evaluation-runner.md | 558 ++++++++++++++++++ 3 files changed, 756 insertions(+), 1 deletion(-) create mode 100644 docs/plans/2026-07-18-h1-live-evaluation-runner-design.md create mode 100644 docs/plans/2026-07-18-h1-live-evaluation-runner.md diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index ca7a40d..bb9e486 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -7,7 +7,7 @@ the canonical authority spec on top of those. - **Repo:** `/Users/ernestprovo/dev/conclave/` - **Version:** 1.1.0 ยท **License:** MIT -- **Last updated:** 2026-07-17 +- **Last updated:** 2026-07-18 --- @@ -34,6 +34,8 @@ the canonical authority spec on top of those. | **H1 Evaluation Plan** | [`docs/plans/2026-07-17-h1-budget-matched-evaluation.md`](docs/plans/2026-07-17-h1-budget-matched-evaluation.md) | TDD delivery plan for the offline, budget-matched evaluation substrate. | | **H1 Method Hardening** | [`docs/plans/2026-07-17-h1-method-hardening-design.md`](docs/plans/2026-07-17-h1-method-hardening-design.md) | Paid-study provenance, task-clustered inference, grader controls, confirmatory refusal boundaries, and QA design. | | **H1 Synthetic QA Pack** | [`studies/elite_qa_v1/README.md`](studies/elite_qa_v1/README.md) | Balanced 24-task open-book harness fixture, QA protocol, and confirmatory preregistration template. | +| **H1 Live Runner Design** | [`docs/plans/2026-07-18-h1-live-evaluation-runner-design.md`](docs/plans/2026-07-18-h1-live-evaluation-runner-design.md) | Sequential paid-exploratory execution, frozen pricing, pessimistic reservations, USD 10 cap, atomic checkpoints, and no-repeat resume. | +| **H1 Live Runner Plan** | [`docs/plans/2026-07-18-h1-live-evaluation-runner.md`](docs/plans/2026-07-18-h1-live-evaluation-runner.md) | Exact TDD tasks for the six live conditions, dry-run estimator, replay fixtures, CLI gate, and correctness-only paid smoke. | --- @@ -114,6 +116,7 @@ Run: `pytest` (config in `pyproject.toml`, `asyncio_mode = "auto"`). | Date | Change | |------|--------| +| 2026-07-18 | Added the validated design and TDD plan for a sequential, checkpointed H1 paid-exploratory runner with frozen pricing and a USD 10 hard cap; implementation and provider calls remain pending. | | 2026-07-17 | Added the experimental DSE-708 offline evaluation substrate and `conclave eval` artifact workflow; no live study or quality claim. | | 2026-07-17 | Reframed the product roadmap around empirically proven decision quality: narrowed current claims to execution traceability, identified answer IDs as internal provenance rather than external evidence, gated Elite merge on H0 correctness, and added H1-H4 evidence, buyer, and outcome gates. | | 2026-07-17 | Documented the implemented-but-unreleased Elite Decision Protocol: fixed three-success gates, initial/claim-audit/revision artifacts, existing final verdict, phased receipts, failure semantics, cost/latency tradeoff, and no streaming. | diff --git a/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md b/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md new file mode 100644 index 0000000..20088e7 --- /dev/null +++ b/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md @@ -0,0 +1,194 @@ +# H1 Live Evaluation Runner Design + +**Linear:** DSE-708 +**Evidence class:** Paid exploratory pilot only +**Approved ceiling:** USD 10.00 +**Boundary:** Verify correctness with minimal paid calls; do not study efficiency or make a +decision-quality claim. + +## Decision + +Add a sequential, checkpointed execution path under `conclave.evals`. The existing +`run_study` matrix remains the cell-level primitive, while an eval-only live provider client +owns every paid call. Before each call, the client computes a pessimistic reservation from a +manifest-bound price snapshot, writes that reservation to an atomic checkpoint, and refuses +the call if committed cost plus the reservation would exceed USD 10.00. It permits exactly +one in-flight provider request. + +This is the smallest approach that makes the cap enforceable. Calling `Council` directly is +not suitable because `fan_out` intentionally launches concurrent calls and does not expose a +pre-call cost gate. A subprocess wrapper would isolate calls but add process, credential, and +checkpoint complexity without improving the accounting invariant. The live runner therefore +reuses the existing adapter/provider highway through `call_model`, but owns condition +orchestration, call order, reservations, and persistence. + +## Components + +- `pricing.py` defines and loads an external, immutable price book. It validates exact + provider/model/revision coverage, canonical hashing, USD currency, positive pessimistic + input/output rates, and the `FrozenStudyDesign.price_snapshot` binding. No current prices + are compiled into library code. +- `live_protocols.py` defines the six versioned condition call graphs and deterministic + per-stage output-token allocation. Calls are awaited serially and all stage caps sum to the + cell's frozen `max_output_tokens`. +- `live.py` owns the guarded provider client, bounded call receipts, atomic checkpoint state, + interruption recovery, dry-run estimation, and final `StudyRun` assembly. +- `eval_cli.py` adds a separate `conclave eval live` surface. Dry-run is the default. Paid + execution requires both `--execute` and an `--approve-spend-usd` value that exactly matches + the frozen manifest ceiling. + +The new path accepts only a complete `paid_exploratory_pilot` manifest with a frozen design. +It rejects legacy synthetic manifests and confirmatory manifests. + +## Frozen price snapshot and reservation + +The operator supplies a JSON price book containing a snapshot ID, capture timestamp, +currency, and one pessimistic rate entry for every exact provider/model/revision in every +roster. The canonical entry hash must equal `FrozenStudyDesign.price_snapshot.prices_hash`; +the snapshot ID, capture time, and currency must also match. Duplicate or missing entries, +unknown models, non-USD currency, and nonpositive rates fail before key resolution or network +access. + +Each call reservation is computed from the frozen call specification, exact public prompt, +fixed prompt-template allowance, upstream stage token ceilings that may be inserted into a +later prompt, provider framing allowance, and the call's output-token ceiling. The price book +uses maximum applicable input and output rates, not discounted or cached rates. Reservation +arithmetic uses `Decimal` and rounds upward to USD microcents. Tests pin the formula. + +Before network I/O: + +1. Acquire the runner's single-call lock. +2. Calculate the reservation from the exact frozen call context. +3. Refuse if `committed_usd + reservation_usd > 10.00`. +4. Atomically persist a pending call with the reservation. +5. Invoke `call_model` with that stage's `max_output_tokens`. + +Afterward, complete usage is priced from provider token receipts. The runner commits the +actual calculated cost when it is within the reservation; missing usage commits the full +reservation. A usage count or calculated cost above the reservation is a fail-closed +`reservation_breach`: the full reservation is charged, the cell is non-successful, and no +new call is scheduled. The final artifacts distinguish usage-priced cost from pessimistically +charged cost. + +## Six conditions + +Every condition receives the same `PublicTask.prompt` and `reference_packets`. Roster order +is already frozen; the first member is the single-frontier and synthesis member, while all +members participate in multi-model stages. Paid manifests must freeze at least three members +per roster so `elite_full` exercises its implemented three-responder contract. + +1. `single_frontier`: the first roster member produces the final answer. +2. `self_refine`: the first member drafts, then revises its own answer. +3. `independent_synthesis`: all members answer independently, then the first member produces + the final synthesis. +4. `critique_only`: all members answer, all surviving members audit the anonymized answer + set, then the first member synthesizes without a revision stage. +5. `revision_only`: all members answer, each sees anonymized peer answers and revises without + a separate claim-audit stage, then the first member synthesizes. +6. `elite_full`: all members answer, audit claims, and revise using the existing versioned + Elite prompt builders; the first member then produces the same synthesis and structured + verdict stages required by the current Elite protocol. + +A frozen allocation table divides each cell ceiling across its stages; integer remainder is +assigned to the final graded-output stage. The allocator rejects a cell ceiling too small to +give every call the minimum useful stage cap. Failed responder gates stop later stages and +produce a failure-inclusive cell record. The graded output is the condition's final decision +artifact, never an internal critique. + +## Data flow + +```text +manifest + public tasks + external price book + | + v + validate hashes and live gates + | + v + dry-run same frozen call graph (no keys/network) + | + --execute + exact $10 approval + | + v + next planned cell -> stage reservation -> atomic pending checkpoint + | | + v v + one call_model await crash-safe evidence + | + v + bounded receipt + checkpoint -> next stage/cell + | + v + complete StudyRun + replay-safe call receipts +``` + +The runner never loads grader keys. Final `StudyRun` records remain compatible with existing +blinding and scoring. A companion live receipt artifact records call IDs, planned-run IDs, +stages, provider/model identities, caps, usage, latency, reservation, charged cost, cost +basis, outcome, and bounded error category. + +## Checkpoint and resume + +The checkpoint is bound to the manifest hash, price-book hash, public-task hash, and USD +10.00 ceiling. It contains completed `RunRecord` values, completed call receipts, committed +cost, and at most one active cell with at most one pending call. It never stores headers, +credential values, endpoint query secrets, raw exceptions, or grader material. + +Every transition is written to a temporary file in the destination directory, flushed, +`fsync`ed, and installed with `os.replace`. Before replacement, serialized content must be +unchanged by the existing secret redactor and must not contain any active provider-key value. + +Resume is deliberately cell-granular. If a process stopped with an active cell, the runner +does not repeat any call from that cell. It charges an unresolved pending reservation at its +full amount, records the cell as `incomplete` with +`interrupted_cell_not_retried`, clears the active state, and continues with the next +unrecorded planned cell. This may waste a small amount of budget, but prevents duplicate +billing and post-crash selection bias. + +When a reservation cannot fit, the runner makes no call and materializes the current and all +remaining cells as `incomplete` with `budget_exhausted`. The matrix stays complete and every +non-executed cell remains in the denominator. + +## Dry-run and replay + +Dry-run traverses the exact same condition call graph without loading configuration, reading +keys, or invoking transport. It reports planned cells, maximum provider calls, worst-case +reserved cost by roster and condition, largest single reservation, total upper bound, the +USD 10.00 ceiling, and whether the complete plan fits. It is an authorization aid, not a bill +forecast. + +Committed test fixtures contain fictional tasks, fake keys supplied only through test +environment variables, a test-only price book, and a sanitized transport replay. The replay +integration executes all six conditions through real adapters with zero network calls, +asserts exact consumption, and produces the same call receipts and `StudyRun` on repeat. + +## Failure modes + +| Failure | Behavior | +|---|---| +| Snapshot, manifest, task, or checkpoint hash drift | Abort before keys or network. | +| Missing/duplicate/unknown price entry | Abort before keys or network. | +| Next reservation crosses USD 10.00 | No call; mark remaining cells `budget_exhausted`. | +| Provider timeout/error/malformed output | Bounded failure receipt; retain cell in denominator. | +| Missing usage | Charge full reservation; mark cost basis pessimistic. | +| Usage exceeds reserved bound | Charge reservation, stop scheduling, record breach. | +| Crash with pending call | Charge full reservation; do not repeat interrupted cell. | +| Checkpoint cannot be persisted | Do not make the next provider call. | +| Secret-like content in checkpoint payload | Reject write and stop. | +| Replay mismatch or unconsumed record | Fail closed; no fallback to network. | + +## Acceptance gates + +- Dry-run and replay make zero network calls and load no live keys. +- The six condition graphs and stage allocations are versioned, deterministic, and complete. +- Tests prove no more than one provider request is active and every request has a persisted + reservation first. +- Total charged cost never exceeds the manifest-bound USD 10.00 ceiling in success, timeout, + missing-usage, budget-exhaustion, and crash/resume tests. +- Resume never repeats an interrupted paid cell. +- Every planned cell appears once in the final `StudyRun`, including unscheduled failures. +- Live/checkpoint/replay artifacts contain no credential values or raw exception chains. +- Focused tests, full pytest, Ruff lint/format, `git diff --check`, and Gitleaks pass. + +This increment ends after a minimal paid correctness smoke test. It does not tune prompts, +compare efficiency, estimate variance, grade model quality, run the 24-task pack at scale, or +support a GO/REDESIGN/KILL conclusion. diff --git a/docs/plans/2026-07-18-h1-live-evaluation-runner.md b/docs/plans/2026-07-18-h1-live-evaluation-runner.md new file mode 100644 index 0000000..3474ea4 --- /dev/null +++ b/docs/plans/2026-07-18-h1-live-evaluation-runner.md @@ -0,0 +1,558 @@ +# H1 Live Evaluation Runner Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add a resumable paid-exploratory runner that executes the six frozen H1 conditions with one provider call in flight and a manifest-bound USD 10.00 hard ceiling. + +**Architecture:** Keep the current failure-inclusive `run_study` contract and add an eval-only sequential call gateway. Bind every pre-call reservation to an external frozen price book, checkpoint before network I/O, and use explicit six-condition executors instead of concurrent `Council.fan_out`. + +**Tech Stack:** Python 3.11, Pydantic v2, Typer, asyncio, Decimal, existing Conclave adapters/`call_model`, pytest/pytest-asyncio, Ruff, Gitleaks. + +--- + +Use @test-driven-development for every production change and +@verification-before-completion before the handoff. Do not call a live provider until Tasks +1-8 pass in replay and the operator has reviewed the dry-run estimate. This plan is paid +exploratory infrastructure only; do not add an efficiency study, quality comparison, or +confirmatory decision gate. + +### Task 1: Frozen price-book contracts and manifest binding + +**Files:** +- Create: `src/conclave/evals/pricing.py` +- Create: `tests/evals/test_pricing.py` +- Create: `tests/fixtures/evals/live_smoke/price_book.json` + +**Step 1: Write the failing contract tests** + +Add tests named: + +- `test_price_book_hash_is_canonical_and_binds_exact_frozen_snapshot` +- `test_price_book_rejects_duplicate_missing_unknown_or_revision_drift` +- `test_price_book_requires_usd_positive_pessimistic_rates` +- `test_call_reservation_rounds_up_and_covers_input_output_and_framing` + +The fixture must use fictional provider/model IDs and rates. Pin a reservation example with +`Decimal`, including prompt-token upper bound, provider framing allowance, inserted upstream +token ceilings, and the stage output cap. Assert no float arithmetic reaches the result. + +**Step 2: Run the focused test and verify RED** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_pricing.py -q +``` + +Expected: FAIL during collection because `conclave.evals.pricing` does not exist. + +**Step 3: Implement the minimal immutable price book** + +In `pricing.py`, add frozen Pydantic contracts for `ModelPrice`, `PriceBook`, and +`CallReservation`; canonical JSON hashing; JSON loading; exact roster coverage validation; +and `reserve_call_cost(...)`. Use ceiling rates per million tokens and quantize upward to +`Decimal("0.000001")`. Validate snapshot ID, capture time, currency, and canonical entry hash +against `FrozenStudyDesign.price_snapshot`. + +Do not add real provider prices to package code or documentation. + +**Step 4: Run focused tests and Ruff** + +Run the Step 2 command, then: + +```bash +python -m ruff check src/conclave/evals/pricing.py tests/evals/test_pricing.py +python -m ruff format --check src/conclave/evals/pricing.py tests/evals/test_pricing.py +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add src/conclave/evals/pricing.py tests/evals/test_pricing.py tests/fixtures/evals/live_smoke/price_book.json +git commit -m "feat(evals): freeze live price snapshots" +``` + +### Task 2: Deterministic six-condition call graphs + +**Files:** +- Create: `src/conclave/evals/live_protocols.py` +- Create: `tests/evals/test_live_protocols.py` +- Modify: `src/conclave/evals/protocols.py` + +**Step 1: Write failing call-graph tests** + +Add tests named: + +- `test_live_registry_covers_exactly_six_versioned_conditions` +- `test_stage_caps_are_positive_deterministic_and_sum_to_cell_ceiling` +- `test_single_and_self_refine_use_only_frozen_lead_member` +- `test_multi_model_conditions_call_members_in_frozen_order` +- `test_critique_revision_and_elite_prompts_are_anonymized` +- `test_elite_uses_current_versioned_prompt_builders_and_three_responder_gate` +- `test_too_small_cell_budget_fails_before_any_call` + +Use a fake sequential client that records `(stage, provider, model, messages, cap)` and +returns deterministic answers. Assert the final output is a decision artifact, not an +internal critique. For a three-member roster, assert exact call order for all six conditions +and assert the recorded maximum concurrency is one. + +**Step 2: Verify RED** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_protocols.py -q +``` + +Expected: FAIL because the live protocol registry and executors are absent. + +**Step 3: Add the minimal versioned protocol implementation** + +Implement a frozen `StageCall` contract, one allocation table, public-task prompt assembly, +and six executor functions. Reuse existing Elite prompt constants/builders for +`elite_full`; do not copy their text. Use the first frozen roster member as lead/synthesizer, +all members in the multi-model stages, and sequential `await` calls only. Reject paid live +rosters with fewer than three members. + +Keep `protocols.py` as the canonical six-ID registry and expose only the live registry hook +needed by the runner. + +**Step 4: Verify GREEN and regress the offline planner** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_protocols.py tests/evals/test_protocols.py tests/test_elite_mode.py -q +python -m ruff check src/conclave/evals/live_protocols.py src/conclave/evals/protocols.py tests/evals/test_live_protocols.py +python -m ruff format --check src/conclave/evals/live_protocols.py src/conclave/evals/protocols.py tests/evals/test_live_protocols.py +``` + +Expected: PASS with existing offline and Elite behavior unchanged. + +**Step 5: Commit** + +```bash +git add src/conclave/evals/live_protocols.py src/conclave/evals/protocols.py tests/evals/test_live_protocols.py +git commit -m "feat(evals): define live condition protocols" +``` + +### Task 3: Guarded provider client and bounded receipts + +**Files:** +- Create: `src/conclave/evals/live.py` +- Create: `tests/evals/test_live_gateway.py` +- Modify: `src/conclave/evals/__init__.py` + +**Step 1: Write failing gateway tests** + +Add tests named: + +- `test_gateway_persists_reservation_before_calling_provider` +- `test_gateway_allows_only_one_in_flight_call` +- `test_gateway_rejects_call_that_would_cross_hard_cap` +- `test_gateway_prices_complete_usage_and_charges_reservation_when_missing` +- `test_gateway_stops_on_reservation_breach` +- `test_gateway_receipt_contains_bounded_error_category_not_raw_exception` + +Inject a fake `call_model` coroutine and checkpoint callback. The fake must assert that a +pending reservation already exists when it starts. Launch two gateway calls with +`asyncio.gather` only to prove the internal lock holds maximum active calls at one. Use caps +small enough to exercise the boundary exactly. + +**Step 2: Verify RED** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_gateway.py -q +``` + +Expected: FAIL because `LiveProviderClient` and receipt contracts are absent. + +**Step 3: Implement the minimal gateway** + +Add frozen `ProviderCallReceipt`, `PendingCall`, and cost-basis contracts plus +`LiveProviderClient`. Resolve price before keys, acquire one `asyncio.Lock`, persist pending +state before `await call_model`, always pass the stage `max_output_tokens`, map errors to the +same bounded categories as provider receipts, and reconcile `ModelAnswer.usage` against the +reservation. Never retain messages, headers, URLs, keys, or raw exception text in receipts. + +**Step 4: Verify GREEN and provider regressions** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_gateway.py tests/test_providers.py tests/test_adapters.py -q +python -m ruff check src/conclave/evals/live.py tests/evals/test_live_gateway.py +python -m ruff format --check src/conclave/evals/live.py tests/evals/test_live_gateway.py +``` + +Expected: PASS; default provider requests remain unchanged. + +**Step 5: Commit** + +```bash +git add src/conclave/evals/live.py src/conclave/evals/__init__.py tests/evals/test_live_gateway.py +git commit -m "feat(evals): guard paid provider calls" +``` + +### Task 4: Atomic checkpoints and no-repeat resume + +**Files:** +- Modify: `src/conclave/evals/live.py` +- Create: `tests/evals/test_live_checkpoint.py` + +**Step 1: Write failing checkpoint tests** + +Add tests named: + +- `test_checkpoint_write_is_flush_fsync_replace_and_secret_scanned` +- `test_checkpoint_rejects_manifest_price_task_or_ceiling_drift` +- `test_resume_charges_pending_reservation_and_never_repeats_interrupted_cell` +- `test_resume_preserves_completed_records_and_call_receipts` +- `test_corrupt_or_tampered_checkpoint_fails_closed` + +Simulate an interruption after pending-call persistence with a `BaseException` so normal +provider-error conversion cannot swallow it. On resume, assert the fake provider never sees +the interrupted planned-run ID, the record is `incomplete`, the deviation code is +`interrupted_cell_not_retried`, and the reservation is included in committed cost. + +Set a fake provider key in the test environment and assert a checkpoint payload containing +that exact value is rejected before `os.replace`. + +**Step 2: Verify RED** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_checkpoint.py -q +``` + +Expected: FAIL because checkpoint load/write/recovery is absent. + +**Step 3: Implement atomic state transitions** + +Add `LiveCheckpoint`, `ActiveCell`, canonical checkpoint hashing, atomic JSON write, strict +load validation, active-key-value detection, and cell-granular recovery. Persist in the +destination directory, flush, `os.fsync`, then `os.replace`. If persistence fails, propagate +before any next provider call. + +**Step 4: Verify GREEN** + +Run the Step 2 command plus: + +```bash +python -m ruff check src/conclave/evals/live.py tests/evals/test_live_checkpoint.py +python -m ruff format --check src/conclave/evals/live.py tests/evals/test_live_checkpoint.py +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add src/conclave/evals/live.py tests/evals/test_live_checkpoint.py +git commit -m "feat(evals): checkpoint live study execution" +``` + +### Task 5: Complete live study execution and USD 10.00 stop + +**Files:** +- Modify: `src/conclave/evals/live.py` +- Modify: `src/conclave/evals/runner.py` +- Create: `tests/evals/test_live_runner.py` + +**Step 1: Write failing end-to-end runner tests** + +Add tests named: + +- `test_live_runner_requires_paid_exploratory_frozen_manifest` +- `test_live_runner_executes_manifest_order_and_builds_complete_study_run` +- `test_budget_exhaustion_makes_no_call_and_marks_all_remaining_cells_incomplete` +- `test_timeout_missing_usage_and_provider_error_never_exceed_ten_dollars` +- `test_final_records_cover_every_planned_run_exactly_once` +- `test_runner_never_loads_grader_keys` + +Use the existing `build_study_manifest` with two three-member fictional rosters and an exact +`approved_spend_ceiling_usd=10.0`. Assert `validate_run_records` accepts the final output and +`total_cost_usd <= 10.0` in every parameterized failure path. + +**Step 2: Verify RED** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_runner.py -q +``` + +Expected: FAIL because live matrix orchestration is absent. + +**Step 3: Implement live matrix orchestration** + +Validate the manifest, tasks, snapshot, checkpoint, evidence class, roster size, and exact +USD 10.00 ceiling before configuration or keys. Execute planned cells in frozen order through +the six-condition registry. Aggregate stage receipts into one `ProtocolExecution`; when the +next reservation does not fit, create `budget_exhausted` incomplete records for every +remaining cell without invoking the gateway. Finalize through existing +`validate_run_records` and the same `StudyRun` accounting fields. + +**Step 4: Verify GREEN and offline-runner compatibility** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_runner.py tests/evals/test_runner.py -q +python -m ruff check src/conclave/evals/live.py src/conclave/evals/runner.py tests/evals/test_live_runner.py +python -m ruff format --check src/conclave/evals/live.py src/conclave/evals/runner.py tests/evals/test_live_runner.py +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add src/conclave/evals/live.py src/conclave/evals/runner.py tests/evals/test_live_runner.py +git commit -m "feat(evals): run capped live study cells" +``` + +### Task 6: Network-free dry-run estimate + +**Files:** +- Modify: `src/conclave/evals/live.py` +- Create: `tests/evals/test_live_estimate.py` + +**Step 1: Write failing estimator tests** + +Add tests named: + +- `test_dry_run_walks_same_calls_without_loading_keys_or_transport` +- `test_dry_run_reports_calls_costs_largest_reservation_and_headroom` +- `test_dry_run_breaks_down_upper_bound_by_roster_and_condition` +- `test_dry_run_rejects_plan_whose_worst_case_exceeds_frozen_ceiling` + +Monkeypatch configuration loading, key lookup, and `transport.post_json` to raise if touched. +Compare estimator call counts and stage caps to the protocol tests, not a duplicate table. + +**Step 2: Verify RED** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_estimate.py -q +``` + +Expected: FAIL because `estimate_live_study` is absent. + +**Step 3: Implement the estimator** + +Traverse the same registry and reservation function used by execution. Return a frozen +`LiveStudyEstimate` with planned cells, maximum call count, per-roster/condition upper bounds, +largest reservation, total upper bound, ceiling, headroom, and `fits_ceiling`. Do not import +or call provider configuration from this path. + +**Step 4: Verify GREEN** + +Run the Step 2 command plus Ruff checks for the changed files. Expected: PASS. + +**Step 5: Commit** + +```bash +git add src/conclave/evals/live.py tests/evals/test_live_estimate.py +git commit -m "feat(evals): estimate capped live studies" +``` + +### Task 7: Fail-closed live CLI + +**Files:** +- Modify: `src/conclave/eval_cli.py` +- Modify: `tests/evals/test_eval_cli.py` + +**Step 1: Write failing CLI tests** + +Add tests named: + +- `test_eval_live_defaults_to_dry_run_and_never_calls_provider` +- `test_eval_live_requires_execute_and_exact_frozen_spend_approval` +- `test_eval_live_rejects_confirmatory_legacy_or_snapshot_drift` +- `test_eval_live_writes_checkpoint_receipts_and_study_run_atomically` +- `test_eval_live_resume_uses_existing_checkpoint` + +The command shape is: + +```bash +conclave eval live MANIFEST TASKS PRICE_BOOK OUTPUT CHECKPOINT RECEIPTS \ + --approve-spend-usd 10.00 [--execute] +``` + +Without `--execute`, print the JSON estimate and exit zero only when it fits. With +`--execute`, require `--approve-spend-usd` to equal both `10.00` and the frozen design value. + +**Step 2: Verify RED** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_eval_cli.py -q +``` + +Expected: FAIL because `eval live` is not registered. + +**Step 3: Implement the CLI adapter** + +Keep CLI behavior thin: load/validate artifacts, call estimator or runner, atomically write +the final run and separate receipt artifact, and render bounded errors. Update the eval app +help so only `live --execute` may reach providers; retain the existing offline `run` +semantics unchanged. + +**Step 4: Verify GREEN and CLI regressions** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_eval_cli.py tests/test_cli.py -q +python -m ruff check src/conclave/eval_cli.py tests/evals/test_eval_cli.py +python -m ruff format --check src/conclave/eval_cli.py tests/evals/test_eval_cli.py +``` + +Expected: PASS. + +**Step 5: Commit** + +```bash +git add src/conclave/eval_cli.py tests/evals/test_eval_cli.py +git commit -m "feat(cli): gate live evaluation execution" +``` + +### Task 8: Sanitized transport replay for all conditions + +**Files:** +- Create: `tests/evals/test_live_replay.py` +- Create: `tests/fixtures/evals/live_smoke/public_tasks.json` +- Create: `tests/fixtures/evals/live_smoke/manifest.json` +- Create: `tests/fixtures/evals/live_smoke/replay.json` +- Modify: `tests/evals/test_replay.py` + +**Step 1: Add the failing replay integration test** + +Add `test_live_smoke_replay_executes_all_conditions_with_zero_network_calls`. Load the +committed artifacts, install `ReplayingPostJson` at the existing transport seam, set only a +fake test key, execute the live runner, call `assert_consumed`, and assert: + +- all six conditions produced exactly one final cell record; +- call order and counts match the frozen live protocol registry; +- the result and call receipts are byte-stable on a second replay; +- the replay, checkpoint, receipts, and run JSON contain no fake key value; +- an extra, missing, or changed request fails closed and never falls back to network. + +**Step 2: Verify RED** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider tests/evals/test_live_replay.py tests/evals/test_replay.py -q +``` + +Expected: FAIL until the sanitized fixtures match the exact request identities. + +**Step 3: Record fixtures with fake credentials and review them** + +Generate the replay only through `RecordingPostJson` backed by a deterministic fake +delegate. Do not call a provider. Inspect the committed JSON and assert the fake credential +is absent before staging it. Keep all task/model/rate data explicitly fictional. + +**Step 4: Verify GREEN** + +Run the Step 2 command, Ruff on the tests, and: + +```bash +rg -n 'sk-|AIza|Bearer|authorization|x-api-key' tests/fixtures/evals/live_smoke +``` + +Expected: tests PASS; the scan finds no credential value or header field. + +**Step 5: Commit** + +```bash +git add tests/evals/test_live_replay.py tests/evals/test_replay.py tests/fixtures/evals/live_smoke +git commit -m "test(evals): replay capped live conditions" +``` + +### Task 9: Documentation, full verification, and paid-smoke handoff + +**Files:** +- Modify: `README.md` +- Modify: `SYSTEM_CONTEXT_DIAGRAM.md` +- Modify: `DOCUMENTATION_INDEX.md` +- Modify: `docs/PRODUCT_DESIGN_DOCUMENT.md` +- Modify: `CHANGELOG.md` + +**Step 1: Write the documentation assertions** + +Extend the existing eval CLI/QA tests to require these exact boundaries in docs: + +- live execution is paid exploratory only; +- dry-run is default and `--execute` plus exact USD 10.00 approval is required; +- one call is in flight, reservations precede calls, and resume never repeats an interrupted + cell; +- the 24-task fixture remains offline/open-book and is not the paid smoke corpus; +- the smoke checks correctness only, not efficiency or decision quality. + +**Step 2: Verify RED** + +Run the new documentation assertion test. Expected: FAIL on the current offline-only text. + +**Step 3: Update the five existing docs** + +Describe the new opt-in live edge honestly, retain the offline `eval run` path, add dry-run +and execute examples using placeholder artifact paths, and keep the PDD and three core docs +under 500 lines. Do not publish real prices, endpoints, task keys, or pilot results. + +**Step 4: Run the complete local gate** + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider -q +python -m ruff check . +python -m ruff format --check . +git diff --check +gitleaks git --redact +``` + +Expected: all tests and static/security checks PASS. If Gitleaks is unavailable, stop and +report the missing verification; do not claim the security gate passed. + +**Step 5: Run the network-free operator rehearsal** + +Run the CLI dry-run and sanitized replay fixtures. Confirm no provider call occurs, the +estimate fits USD 10.00, the checkpoint/resume path passes, and output labels say paid +exploratory/not decision eligible. + +**Step 6: Self-audit the plan boundary before any paid call** + +Confirm the branch contains no efficiency analysis, no quality claim, no confirmatory gate +change, no grader-key access, no mutable price service, and no concurrency above one. Fix any +drift before proceeding. + +**Step 7: Commit** + +```bash +git add README.md SYSTEM_CONTEXT_DIAGRAM.md DOCUMENTATION_INDEX.md docs/PRODUCT_DESIGN_DOCUMENT.md CHANGELOG.md tests +git commit -m "docs(evals): document capped live pilot" +``` + +**Step 8: Paid smoke execution gate** + +Only after Tasks 1-9 are reviewed, run the dry-run against the separately prepared paid-smoke +manifest and price book. Execute the smallest predeclared matrix that exercises all six +conditions under the shared USD 10.00 ceiling. Stop after correctness artifacts are verified; +do not expand calls to improve precision or compare efficiency. + +**Step 9: Final handoff evidence** + +Provide commit SHA, exact test/lint/format/diff/Gitleaks results, dry-run estimate, charged +cost at or below USD 10.00, checkpoint/resume evidence, sanitized artifact paths, and any +blocked provider/model cells. Do not present exploratory outputs as proof that Elite is +better. From a7116e855585ff341b7ba252d2337d55459b1be9 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 20:25:23 -0400 Subject: [PATCH 02/14] feat(evals): freeze live price snapshots --- src/conclave/evals/pricing.py | 249 ++++++++++++++++++ tests/evals/test_pricing.py | 214 +++++++++++++++ .../fixtures/evals/live_smoke/price_book.json | 32 +++ 3 files changed, 495 insertions(+) create mode 100644 src/conclave/evals/pricing.py create mode 100644 tests/evals/test_pricing.py create mode 100644 tests/fixtures/evals/live_smoke/price_book.json diff --git a/src/conclave/evals/pricing.py b/src/conclave/evals/pricing.py new file mode 100644 index 0000000..de3c72c --- /dev/null +++ b/src/conclave/evals/pricing.py @@ -0,0 +1,249 @@ +"""Frozen external price snapshots and pessimistic call reservations.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable, Sequence +from decimal import ROUND_CEILING, Decimal, localcontext +from pathlib import Path +from typing import Literal + +from pydantic import Field, field_validator, model_validator + +from .models import EvalModel, FrozenStudyDesign + +USD_MICROCENT = Decimal("0.000001") +TOKENS_PER_MILLION = Decimal(1_000_000) +_PRICE_HASH_NAMESPACE = "conclave_model_prices_v1" +_ModelIdentity = tuple[str, str, str] + + +class ModelPrice(EvalModel): + """Pessimistic ceiling rates for one exact provider model revision.""" + + provider_id: str = Field(min_length=1) + model_id: str = Field(min_length=1) + model_revision: str = Field(min_length=1) + input_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) + output_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) + + @field_validator( + "input_ceiling_usd_per_million_tokens", + "output_ceiling_usd_per_million_tokens", + mode="before", + ) + @classmethod + def require_exact_decimal_rate(cls, value: object) -> object: + if isinstance(value, (bool, float)): + raise ValueError("price rates must be exact decimal values, not floats") + return value + + @property + def identity(self) -> _ModelIdentity: + return (self.provider_id, self.model_id, self.model_revision) + + +class PriceBook(EvalModel): + """One immutable external snapshot covering a frozen study roster.""" + + snapshot_id: str = Field(min_length=1) + captured_at: str = Field(min_length=1) + currency: Literal["USD"] + entries: tuple[ModelPrice, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_unique_entries(self) -> PriceBook: + identities = [entry.identity for entry in self.entries] + if len(set(identities)) != len(identities): + raise ValueError("price book provider/model/revision identities must be unique") + return self + + +class CallReservation(EvalModel): + """Auditable worst-case token and cost bounds for one provider call.""" + + provider_id: str = Field(min_length=1) + model_id: str = Field(min_length=1) + model_revision: str = Field(min_length=1) + prompt_token_upper_bound: int = Field(ge=0) + prompt_template_token_allowance: int = Field(ge=0) + provider_framing_token_allowance: int = Field(ge=0) + upstream_output_token_ceilings: tuple[int, ...] + input_token_upper_bound: int = Field(ge=0) + output_token_upper_bound: int = Field(gt=0) + input_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) + output_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) + input_cost_upper_bound_usd: Decimal = Field(ge=0) + output_cost_upper_bound_usd: Decimal = Field(gt=0) + reserved_cost_usd: Decimal = Field(gt=0) + + +def _canonical_decimal(value: Decimal) -> str: + sign, digits, exponent = value.as_tuple() + digit_text = "".join(str(digit) for digit in digits) + if exponent >= 0: + integer = digit_text + ("0" * exponent) + fraction = "" + else: + split_at = len(digit_text) + exponent + if split_at > 0: + integer = digit_text[:split_at] + fraction = digit_text[split_at:] + else: + integer = "0" + fraction = ("0" * -split_at) + digit_text + fraction = fraction.rstrip("0") + canonical = integer if not fraction else f"{integer}.{fraction}" + return f"-{canonical}" if sign else canonical + + +def _canonical_entry(entry: ModelPrice) -> dict[str, str]: + return { + "schema_version": entry.schema_version, + "provider_id": entry.provider_id, + "model_id": entry.model_id, + "model_revision": entry.model_revision, + "input_ceiling_usd_per_million_tokens": _canonical_decimal( + entry.input_ceiling_usd_per_million_tokens + ), + "output_ceiling_usd_per_million_tokens": _canonical_decimal( + entry.output_ceiling_usd_per_million_tokens + ), + } + + +def hash_price_entries(entries: Iterable[ModelPrice]) -> str: + """Return an order- and representation-independent digest of model rates.""" + + ordered = sorted(entries, key=lambda entry: entry.identity) + canonical = json.dumps( + { + "namespace": _PRICE_HASH_NAMESPACE, + "entries": [_canonical_entry(entry) for entry in ordered], + }, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" + + +def _format_identities(identities: set[_ModelIdentity]) -> str: + return ", ".join("/".join(identity) for identity in sorted(identities)) + + +def validate_price_book(price_book: PriceBook, *, frozen_design: FrozenStudyDesign) -> None: + """Validate exact snapshot metadata, roster coverage, and canonical entry hash.""" + + snapshot = frozen_design.price_snapshot + if snapshot.snapshot_id != price_book.snapshot_id: + raise ValueError("price book snapshot_id does not match frozen design") + if snapshot.captured_at != price_book.captured_at: + raise ValueError("price book captured_at does not match frozen design") + if snapshot.currency != "USD" or snapshot.currency != price_book.currency: + raise ValueError("price book currency must be USD and match frozen design") + + expected = { + (member.provider_id, member.model_id, member.model_revision) + for roster in frozen_design.rosters + for member in roster.members + } + actual = {entry.identity for entry in price_book.entries} + missing = expected - actual + unknown = actual - expected + if missing or unknown: + parts = [] + if missing: + parts.append(f"missing={_format_identities(missing)}") + if unknown: + parts.append(f"unknown={_format_identities(unknown)}") + raise ValueError(f"price book roster coverage mismatch: {'; '.join(parts)}") + + if hash_price_entries(price_book.entries) != snapshot.prices_hash: + raise ValueError("price book prices_hash does not match frozen design") + + +def load_price_book(path: str | Path, *, frozen_design: FrozenStudyDesign) -> PriceBook: + """Load and bind an external JSON price book to one frozen study design.""" + + with Path(path).open(encoding="utf-8") as handle: + payload = json.load(handle, parse_float=Decimal) + price_book = PriceBook.model_validate(payload) + validate_price_book(price_book, frozen_design=frozen_design) + return price_book + + +def _validate_token_bound(name: str, value: int, *, positive: bool = False) -> None: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + minimum = 1 if positive else 0 + if value < minimum: + comparator = "positive" if positive else "nonnegative" + raise ValueError(f"{name} must be {comparator}") + + +def reserve_call_cost( + price: ModelPrice, + *, + prompt_token_upper_bound: int, + prompt_template_token_allowance: int, + provider_framing_token_allowance: int, + upstream_output_token_ceilings: Sequence[int], + max_output_tokens: int, +) -> CallReservation: + """Reserve the pessimistic USD cost of all possible input and output tokens.""" + + _validate_token_bound("prompt_token_upper_bound", prompt_token_upper_bound) + _validate_token_bound("prompt_template_token_allowance", prompt_template_token_allowance) + _validate_token_bound("provider_framing_token_allowance", provider_framing_token_allowance) + _validate_token_bound("max_output_tokens", max_output_tokens, positive=True) + upstream_ceilings = tuple(upstream_output_token_ceilings) + for index, ceiling in enumerate(upstream_ceilings): + _validate_token_bound(f"upstream_output_token_ceilings[{index}]", ceiling) + + input_token_upper_bound = ( + prompt_token_upper_bound + + prompt_template_token_allowance + + provider_framing_token_allowance + + sum(upstream_ceilings) + ) + precision = max( + 64, + len(str(input_token_upper_bound)) + + len(price.input_ceiling_usd_per_million_tokens.as_tuple().digits) + + 20, + len(str(max_output_tokens)) + + len(price.output_ceiling_usd_per_million_tokens.as_tuple().digits) + + 20, + ) + with localcontext() as context: + context.prec = precision + input_cost = ( + Decimal(input_token_upper_bound) + * price.input_ceiling_usd_per_million_tokens + / TOKENS_PER_MILLION + ) + output_cost = ( + Decimal(max_output_tokens) + * price.output_ceiling_usd_per_million_tokens + / TOKENS_PER_MILLION + ) + reserved_cost = (input_cost + output_cost).quantize(USD_MICROCENT, rounding=ROUND_CEILING) + + return CallReservation( + provider_id=price.provider_id, + model_id=price.model_id, + model_revision=price.model_revision, + prompt_token_upper_bound=prompt_token_upper_bound, + prompt_template_token_allowance=prompt_template_token_allowance, + provider_framing_token_allowance=provider_framing_token_allowance, + upstream_output_token_ceilings=upstream_ceilings, + input_token_upper_bound=input_token_upper_bound, + output_token_upper_bound=max_output_tokens, + input_ceiling_usd_per_million_tokens=price.input_ceiling_usd_per_million_tokens, + output_ceiling_usd_per_million_tokens=price.output_ceiling_usd_per_million_tokens, + input_cost_upper_bound_usd=input_cost, + output_cost_upper_bound_usd=output_cost, + reserved_cost_usd=reserved_cost, + ) diff --git a/tests/evals/test_pricing.py b/tests/evals/test_pricing.py new file mode 100644 index 0000000..9916408 --- /dev/null +++ b/tests/evals/test_pricing.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import json +from decimal import Decimal +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from conclave.evals.models import ( + AnalysisGateConfig, + BootstrapConfig, + ExclusionDeviationPolicy, + FrozenStudyDesign, + PriceSnapshot, + ProviderModelSpec, + RandomizationConfig, + RosterSpec, + TimeoutRetryPolicy, +) +from conclave.evals.pricing import ( + CallReservation, + PriceBook, + hash_price_entries, + load_price_book, + reserve_call_cost, + validate_price_book, +) +from conclave.evals.protocols import CONDITION_IDS + +DIGEST = "sha256:" + "a" * 64 +PRICE_FIXTURE = Path(__file__).parents[1] / "fixtures/evals/live_smoke/price_book.json" +EXPECTED_PRICE_HASH = "sha256:df85e17c12985ada8a350c6f2adc1778a0218bf3dc70cf3204ac8d1057939aac" + + +def _payload() -> dict[str, object]: + return json.loads(PRICE_FIXTURE.read_text(encoding="utf-8")) + + +def _book(payload: dict[str, object] | None = None) -> PriceBook: + return PriceBook.model_validate(payload or _payload()) + + +def _design(*, prices_hash: str, snapshot_updates: dict[str, str] | None = None): + snapshot = { + "snapshot_id": "fictional-live-smoke-prices-2026-07-18", + "captured_at": "2026-07-18T12:00:00Z", + "currency": "USD", + "prices_hash": prices_hash, + } + snapshot.update(snapshot_updates or {}) + rosters = ( + RosterSpec( + roster_id="fictional-roster-a", + members=( + ProviderModelSpec( + provider_id="fictional-provider-a", + model_id="fictional-model-a", + model_revision="fixture-r1", + ), + ProviderModelSpec( + provider_id="fictional-provider-b", + model_id="fictional-model-b", + model_revision="fixture-r2", + ), + ), + ), + RosterSpec( + roster_id="fictional-roster-b", + members=( + ProviderModelSpec( + provider_id="fictional-provider-c", + model_id="fictional-model-c", + model_revision="fixture-r3", + ), + ), + ), + ) + return FrozenStudyDesign( + evidence_classification="paid_exploratory_pilot", + base_commit="1" * 40, + task_family_map={"fictional-task": "fixture-family"}, + rosters=rosters, + condition_prompt_versions={condition: "prompt-v1" for condition in CONDITION_IDS}, + condition_protocol_versions={condition: "protocol-v1" for condition in CONDITION_IDS}, + generation_settings_hash=DIGEST, + evaluator_version="evaluator-v1", + analysis_code_hash=DIGEST, + rubric_hash=DIGEST, + grader_instructions_hash=DIGEST, + grader_keys_hash=DIGEST, + exclusion_deviation_policy=ExclusionDeviationPolicy(), + timeout_retry_policy=TimeoutRetryPolicy(timeout_seconds=30, retry_attempts=0), + randomization=RandomizationConfig(master_seed=20260718), + bootstrap=BootstrapConfig(seed=20260718, samples=10), + analysis_gates=AnalysisGateConfig( + primary_baseline="single_frontier", + absolute_p95_latency_seconds=60, + minimum_confirmatory_tasks=2, + ), + price_snapshot=PriceSnapshot(**snapshot), + approved_spend_ceiling_usd=10, + ) + + +def test_price_book_hash_is_canonical_and_binds_exact_frozen_snapshot() -> None: + book = _book() + price_hash = hash_price_entries(book.entries) + + assert price_hash == EXPECTED_PRICE_HASH + assert hash_price_entries(reversed(book.entries)) == price_hash + first_precise = book.entries[0].model_copy( + update={"input_ceiling_usd_per_million_tokens": Decimal("1.12345678901234567890123456781")} + ) + second_precise = first_precise.model_copy( + update={"input_ceiling_usd_per_million_tokens": Decimal("1.12345678901234567890123456782")} + ) + assert hash_price_entries((first_precise, *book.entries[1:])) != hash_price_entries( + (second_precise, *book.entries[1:]) + ) + assert load_price_book(PRICE_FIXTURE, frozen_design=_design(prices_hash=price_hash)) == book + with pytest.raises(ValidationError): + book.snapshot_id = "mutated" + + for field, changed, error in ( + ("snapshot_id", "other-snapshot", "snapshot_id"), + ("captured_at", "2026-07-18T12:00:01Z", "captured_at"), + ("prices_hash", DIGEST, "prices_hash"), + ): + design = _design(prices_hash=price_hash, snapshot_updates={field: changed}) + with pytest.raises(ValueError, match=error): + validate_price_book(book, frozen_design=design) + + +def test_price_book_rejects_duplicate_missing_unknown_or_revision_drift() -> None: + payload = _payload() + entries = payload["entries"] + assert isinstance(entries, list) + entries.append(dict(entries[0])) + with pytest.raises(ValidationError, match="unique"): + _book(payload) + + complete = _book() + design = _design(prices_hash=hash_price_entries(complete.entries)) + + missing = complete.model_copy(update={"entries": complete.entries[:-1]}) + with pytest.raises(ValueError, match="missing"): + validate_price_book(missing, frozen_design=design) + + unknown_entry = complete.entries[0].model_copy( + update={ + "provider_id": "fictional-provider-unknown", + "model_id": "fictional-model-unknown", + } + ) + unknown = complete.model_copy(update={"entries": (*complete.entries, unknown_entry)}) + with pytest.raises(ValueError, match="unknown"): + validate_price_book(unknown, frozen_design=design) + + drifted_entry = complete.entries[0].model_copy(update={"model_revision": "fixture-r99"}) + drifted = complete.model_copy(update={"entries": (drifted_entry, *complete.entries[1:])}) + with pytest.raises(ValueError, match="missing.*unknown"): + validate_price_book(drifted, frozen_design=design) + + +def test_price_book_requires_usd_positive_pessimistic_rates() -> None: + payload = _payload() + payload["currency"] = "EUR" + with pytest.raises(ValidationError, match="USD"): + _book(payload) + + for field in ( + "input_ceiling_usd_per_million_tokens", + "output_ceiling_usd_per_million_tokens", + ): + for invalid in ("0", "-0.000001"): + payload = _payload() + entries = payload["entries"] + assert isinstance(entries, list) + entries[0][field] = invalid + with pytest.raises(ValidationError, match="greater than 0"): + _book(payload) + + +def test_call_reservation_rounds_up_and_covers_input_output_and_framing() -> None: + price = _book().entries[0] + + reservation = reserve_call_cost( + price, + prompt_token_upper_bound=101, + prompt_template_token_allowance=17, + provider_framing_token_allowance=11, + upstream_output_token_ceilings=(50, 60), + max_output_tokens=75, + ) + + assert isinstance(reservation, CallReservation) + assert reservation.input_token_upper_bound == 239 + assert reservation.output_token_upper_bound == 75 + assert reservation.input_cost_upper_bound_usd == Decimal("0.000295061513") + assert reservation.output_cost_upper_bound_usd == Decimal("0.000342591825") + assert reservation.reserved_cost_usd == Decimal("0.000638") + assert all( + type(value) is Decimal + for value in ( + reservation.input_ceiling_usd_per_million_tokens, + reservation.output_ceiling_usd_per_million_tokens, + reservation.input_cost_upper_bound_usd, + reservation.output_cost_upper_bound_usd, + reservation.reserved_cost_usd, + ) + ) + with pytest.raises(ValidationError): + reservation.reserved_cost_usd = Decimal("0") diff --git a/tests/fixtures/evals/live_smoke/price_book.json b/tests/fixtures/evals/live_smoke/price_book.json new file mode 100644 index 0000000..a9bedb5 --- /dev/null +++ b/tests/fixtures/evals/live_smoke/price_book.json @@ -0,0 +1,32 @@ +{ + "schema_version": "conclave_eval_v1", + "snapshot_id": "fictional-live-smoke-prices-2026-07-18", + "captured_at": "2026-07-18T12:00:00Z", + "currency": "USD", + "entries": [ + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-a", + "model_id": "fictional-model-a", + "model_revision": "fixture-r1", + "input_ceiling_usd_per_million_tokens": "1.234567", + "output_ceiling_usd_per_million_tokens": "4.567891" + }, + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-b", + "model_id": "fictional-model-b", + "model_revision": "fixture-r2", + "input_ceiling_usd_per_million_tokens": "2.000001", + "output_ceiling_usd_per_million_tokens": "6.000001" + }, + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-c", + "model_id": "fictional-model-c", + "model_revision": "fixture-r3", + "input_ceiling_usd_per_million_tokens": "3.000001", + "output_ceiling_usd_per_million_tokens": "9.000001" + } + ] +} From c38317545871d8232c4cee309669d029ee8b1847 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 20:32:47 -0400 Subject: [PATCH 03/14] feat(evals): define live condition protocols --- src/conclave/evals/live_protocols.py | 504 +++++++++++++++++++++++++++ src/conclave/evals/protocols.py | 12 + tests/evals/test_live_protocols.py | 260 ++++++++++++++ 3 files changed, 776 insertions(+) create mode 100644 src/conclave/evals/live_protocols.py create mode 100644 tests/evals/test_live_protocols.py diff --git a/src/conclave/evals/live_protocols.py b/src/conclave/evals/live_protocols.py new file mode 100644 index 0000000..1697068 --- /dev/null +++ b/src/conclave/evals/live_protocols.py @@ -0,0 +1,504 @@ +"""Sequential, versioned call graphs for the six paid evaluation conditions.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Literal, Protocol + +from pydantic import Field + +from conclave import prompts +from conclave.council import _SYNTH_SYSTEM +from conclave.models import ELITE_MIN_RESPONDERS, ModelAnswer, derive_phase_answer_id +from conclave.verdict import VERDICT_EXTRACTION_PROMPT_VERSION +from conclave.verdict_synthesis import _build_messages as _verdict_messages + +from .models import ( + EVAL_CONDITION_IDS, + ConditionId, + EvalModel, + ProviderModelSpec, + PublicTask, +) + +StageName = Literal[ + "initial", + "draft", + "self_revision", + "critique", + "revision", + "synthesis", + "verdict", +] + + +class ChatMessage(EvalModel): + """One immutable OpenAI-style prompt message.""" + + role: Literal["system", "user"] + content: str = Field(min_length=1) + + +class StageCall(EvalModel): + """One fully resolved, sequential provider call in a live condition.""" + + stage: StageName + provider_id: str = Field(min_length=1) + model_id: str = Field(min_length=1) + model_revision: str = Field(min_length=1) + messages: tuple[ChatMessage, ...] = Field(min_length=1) + max_output_tokens: int = Field(gt=0) + upstream_output_token_ceilings: tuple[int, ...] = () + + +class SequentialStageClient(Protocol): + """Narrow client contract implemented by the guarded live gateway.""" + + def call(self, call: StageCall) -> Awaitable[ModelAnswer]: ... + + +@dataclass(frozen=True) +class LiveProtocolSpec: + """Version metadata for one frozen live condition graph.""" + + condition_id: ConditionId + protocol_version: str + prompt_version: str + executor: Callable[[_Execution, str], Awaitable[str]] + + +_STAGE_ALLOCATION_TABLE: Mapping[ + ConditionId, tuple[tuple[StageName, Literal["lead", "all"]], ...] +] = MappingProxyType( + { + "single_frontier": (("initial", "lead"),), + "self_refine": (("draft", "lead"), ("self_revision", "lead")), + "independent_synthesis": (("initial", "all"), ("synthesis", "lead")), + "critique_only": ( + ("initial", "all"), + ("critique", "all"), + ("synthesis", "lead"), + ), + "revision_only": ( + ("initial", "all"), + ("revision", "all"), + ("synthesis", "lead"), + ), + "elite_full": ( + ("initial", "all"), + ("critique", "all"), + ("revision", "all"), + ("synthesis", "lead"), + ("verdict", "lead"), + ), + } +) + +LIVE_PROMPT_VERSIONS: Mapping[ConditionId, str] = MappingProxyType( + { + "single_frontier": "live_single_frontier_prompt_v1", + "self_refine": "live_self_refine_prompt_v1", + "independent_synthesis": f"live_synthesis_{prompts.SYNTHESIS_PROMPT_VERSION}", + "critique_only": ( + f"live_critique_{prompts.ELITE_PROMPT_VERSION}_synthesis_" + f"{prompts.SYNTHESIS_PROMPT_VERSION}" + ), + "revision_only": (f"live_revision_v1_synthesis_{prompts.SYNTHESIS_PROMPT_VERSION}"), + "elite_full": ( + f"elite_{prompts.ELITE_PROMPT_VERSION}_synthesis_" + f"{prompts.SYNTHESIS_PROMPT_VERSION}_verdict_{VERDICT_EXTRACTION_PROMPT_VERSION}" + ), + } +) + + +def stage_call_sequence( + condition_id: ConditionId, *, roster_size: int +) -> tuple[tuple[StageName, int], ...]: + """Expand a condition's fixed stage layout into stage/member call positions.""" + + if isinstance(roster_size, bool) or not isinstance(roster_size, int) or roster_size < 1: + raise ValueError("roster_size must be a positive integer") + return tuple( + (stage, member_index) + for stage, participants in _STAGE_ALLOCATION_TABLE[condition_id] + for member_index in (range(roster_size) if participants == "all" else range(1)) + ) + + +def allocate_stage_caps( + condition_id: ConditionId, *, roster_size: int, cell_ceiling: int +) -> tuple[int, ...]: + """Allocate positive equal caps, assigning integer remainder to graded output.""" + + if isinstance(cell_ceiling, bool) or not isinstance(cell_ceiling, int): + raise TypeError("cell_ceiling must be an integer") + call_count = len(stage_call_sequence(condition_id, roster_size=roster_size)) + if cell_ceiling < call_count: + raise ValueError( + f"cell ceiling {cell_ceiling} is too small for {call_count} positive stage caps" + ) + per_call, remainder = divmod(cell_ceiling, call_count) + return (*([per_call] * (call_count - 1)), per_call + remainder) + + +def _public_prompt(task: PublicTask) -> str: + packets = "\n\n".join( + f"### Reference packet {index}\n{packet}" + for index, packet in enumerate(task.reference_packets, start=1) + ) + return ( + task.prompt if not packets else f"{task.prompt}\n\nPublic reference packets:\n\n{packets}" + ) + + +def _messages(user: str, *, system: str | None = None) -> tuple[ChatMessage, ...]: + items = ([ChatMessage(role="system", content=system)] if system else []) + [ + ChatMessage(role="user", content=user) + ] + return tuple(items) + + +def _anonymized( + answer: ModelAnswer, *, phase: str, index: int, parents: Sequence[str] = () +) -> ModelAnswer: + artifact = answer.model_copy( + update={"name": f"Model {prompts.LETTERS[index]}", "model_id": "anonymized"} + ) + return derive_phase_answer_id(artifact, phase, parent_answer_ids=parents) + + +def _successful( + answers: Sequence[tuple[int, ModelAnswer]], *, phase: str +) -> list[tuple[int, ModelAnswer]]: + return [ + (index, _anonymized(answer, phase=phase, index=index)) + for index, answer in answers + if answer.ok and answer.answer and answer.answer.strip() + ] + + +def _synthesis_user( + public_prompt: str, + artifacts: Sequence[ModelAnswer], + *, + audits: Sequence[ModelAnswer] = (), +) -> str: + panel = "\n\n".join( + f"### Model {prompts.LETTERS[index]} decision artifact\n{artifact.answer}" + for index, artifact in enumerate(artifacts) + ) + audit_panel = "\n\n".join( + f"### Model {prompts.LETTERS[index]} anonymized audit\n{artifact.answer}" + for index, artifact in enumerate(audits) + ) + audit_section = f"\n\nAnonymized claim audits:\n\n{audit_panel}" if audit_panel else "" + return ( + f"Original prompt and public material:\n{public_prompt}\n\n" + f"Anonymized decision artifacts:\n\n{panel}{audit_section}\n\n" + "Produce the consolidated decision artifact." + ) + + +def _revision_only_user( + public_prompt: str, + original: ModelAnswer, + panel: Sequence[ModelAnswer], +) -> str: + peer_panel = "\n\n".join( + f"### Model {prompts.LETTERS[index]} peer answer\n{artifact.answer}" + for index, artifact in enumerate(panel) + ) + return ( + f"Original prompt and public material:\n{public_prompt}\n\n" + f"Your original answer:\n{original.answer}\n\n" + f"Anonymized peer panel:\n\n{peer_panel}\n\n" + "Produce one complete revised answer to the original prompt." + ) + + +class _Execution: + def __init__( + self, + condition_id: ConditionId, + roster: Sequence[ProviderModelSpec], + caps: Sequence[int], + client: SequentialStageClient, + ) -> None: + sequence = stage_call_sequence(condition_id, roster_size=len(roster)) + self._caps = dict(zip(sequence, caps, strict=True)) + self.roster = tuple(roster) + self.client = client + + async def call( + self, + stage: StageName, + member_index: int, + messages: tuple[ChatMessage, ...], + *, + upstream: Sequence[int] = (), + ) -> ModelAnswer: + member = self.roster[member_index] + return await self.client.call( + StageCall( + stage=stage, + provider_id=member.provider_id, + model_id=member.model_id, + model_revision=member.model_revision, + messages=messages, + max_output_tokens=self._caps[(stage, member_index)], + upstream_output_token_ceilings=tuple(upstream), + ) + ) + + async def all( + self, + stage: StageName, + messages_for: Callable[[int], tuple[ChatMessage, ...]], + *, + members: Sequence[int] | None = None, + upstream: Sequence[int] = (), + ) -> list[tuple[int, ModelAnswer]]: + indexes = tuple(range(len(self.roster))) if members is None else tuple(members) + return [ + (index, await self.call(stage, index, messages_for(index), upstream=upstream)) + for index in indexes + ] + + def caps(self, stage: StageName, members: Sequence[int]) -> tuple[int, ...]: + return tuple(self._caps[(stage, index)] for index in members) + + +def _required(answer: ModelAnswer, stage: str) -> str: + if not answer.ok or not answer.answer or not answer.answer.strip(): + raise ValueError(f"{stage} stage did not produce a decision artifact") + return answer.answer + + +def _elite_gate( + phase: str, answers: Sequence[tuple[int, ModelAnswer]] +) -> list[tuple[int, ModelAnswer]]: + successful = _successful(answers, phase=phase) + if len(successful) < ELITE_MIN_RESPONDERS: + raise ValueError( + f"{phase} phase required {ELITE_MIN_RESPONDERS} successful responders; " + f"got {len(successful)}" + ) + return successful + + +async def _initial_panel(run: _Execution, public_prompt: str) -> list[tuple[int, ModelAnswer]]: + answers = await run.all("initial", lambda _index: _messages(public_prompt)) + successful = _successful(answers, phase="initial") + if not successful: + raise ValueError("initial phase produced no successful responders") + return successful + + +async def execute_single_frontier(run: _Execution, public_prompt: str) -> str: + answer = await run.call("initial", 0, _messages(public_prompt)) + return _required(answer, "initial") + + +async def execute_self_refine(run: _Execution, public_prompt: str) -> str: + draft = await run.call("draft", 0, _messages(public_prompt)) + draft_text = _required(draft, "draft") + revision = await run.call( + "self_revision", + 0, + _messages( + f"Original prompt and public material:\n{public_prompt}\n\n" + f"Your draft:\n{draft_text}\n\nProduce one complete, improved final answer." + ), + upstream=run.caps("draft", (0,)), + ) + return _required(revision, "self_revision") + + +async def execute_independent_synthesis(run: _Execution, public_prompt: str) -> str: + initial = await _initial_panel(run, public_prompt) + artifacts = [answer for _index, answer in initial] + synthesis = await run.call( + "synthesis", + 0, + _messages(_synthesis_user(public_prompt, artifacts), system=_SYNTH_SYSTEM), + upstream=run.caps("initial", [index for index, _answer in initial]), + ) + return _required(synthesis, "synthesis") + + +async def execute_critique_only(run: _Execution, public_prompt: str) -> str: + initial = await _initial_panel(run, public_prompt) + members = [index for index, _answer in initial] + artifacts = [answer for _index, answer in initial] + critique_messages = _messages( + prompts.elite_critic_user(public_prompt, artifacts), + system=prompts.ELITE_CRITIC_SYSTEM, + ) + critiques = _successful( + await run.all( + "critique", + lambda _index: critique_messages, + members=members, + upstream=run.caps("initial", members), + ), + phase="critique", + ) + synthesis = await run.call( + "synthesis", + 0, + _messages( + _synthesis_user( + public_prompt, + artifacts, + audits=[answer for _index, answer in critiques], + ), + system=_SYNTH_SYSTEM, + ), + upstream=( + *run.caps("initial", members), + *run.caps("critique", [index for index, _answer in critiques]), + ), + ) + return _required(synthesis, "synthesis") + + +async def execute_revision_only(run: _Execution, public_prompt: str) -> str: + initial = await _initial_panel(run, public_prompt) + members = [index for index, _answer in initial] + by_member = dict(initial) + artifacts = [answer for _index, answer in initial] + revisions = _successful( + await run.all( + "revision", + lambda index: _messages( + _revision_only_user(public_prompt, by_member[index], artifacts) + ), + members=members, + upstream=run.caps("initial", members), + ), + phase="revision", + ) + revision_artifacts = [answer for _index, answer in revisions] + synthesis = await run.call( + "synthesis", + 0, + _messages(_synthesis_user(public_prompt, revision_artifacts), system=_SYNTH_SYSTEM), + upstream=run.caps("revision", [index for index, _answer in revisions]), + ) + return _required(synthesis, "synthesis") + + +async def execute_elite_full(run: _Execution, public_prompt: str) -> str: + initial_raw = await run.all("initial", lambda _index: _messages(public_prompt)) + initial = _elite_gate("initial", initial_raw) + members = [index for index, _answer in initial] + initial_artifacts = [answer for _index, answer in initial] + critique_messages = _messages( + prompts.elite_critic_user(public_prompt, initial_artifacts), + system=prompts.ELITE_CRITIC_SYSTEM, + ) + critique_raw = await run.all( + "critique", + lambda _index: critique_messages, + members=members, + upstream=run.caps("initial", members), + ) + critiques = _elite_gate("critique", critique_raw) + critique_members = [index for index, _answer in critiques] + critique_artifacts = [answer for _index, answer in critiques] + initial_by_member = dict(initial) + parent_ids = tuple( + answer.answer_id or "" for answer in [*initial_artifacts, *critique_artifacts] + ) + revision_raw = await run.all( + "revision", + lambda index: _messages( + prompts.elite_revision_user( + public_prompt, + initial_by_member[index], + initial_artifacts, + critique_artifacts, + ), + system=prompts.ELITE_REVISION_SYSTEM, + ), + members=critique_members, + upstream=( + *run.caps("initial", members), + *run.caps("critique", critique_members), + ), + ) + revisions = _elite_gate("revision", revision_raw) + revision_artifacts = [ + ( + index, + _anonymized(answer, phase="revision", index=index, parents=parent_ids), + ) + for index, answer in revision_raw + if answer.ok and answer.answer and answer.answer.strip() + ] + revision_answers = [answer for _index, answer in revision_artifacts] + synthesis = await run.call( + "synthesis", + 0, + _messages(_synthesis_user(public_prompt, revision_answers), system=_SYNTH_SYSTEM), + upstream=run.caps("revision", [index for index, _answer in revisions]), + ) + _required(synthesis, "synthesis") + verdict = await run.call( + "verdict", + 0, + tuple( + ChatMessage.model_validate(message) + for message in _verdict_messages(public_prompt, revision_answers) + ), + upstream=run.caps("revision", [index for index, _answer in revisions]), + ) + return _required(verdict, "verdict") + + +_EXECUTORS: Mapping[ConditionId, Callable[[_Execution, str], Awaitable[str]]] = MappingProxyType( + { + "single_frontier": execute_single_frontier, + "self_refine": execute_self_refine, + "independent_synthesis": execute_independent_synthesis, + "critique_only": execute_critique_only, + "revision_only": execute_revision_only, + "elite_full": execute_elite_full, + } +) + +LIVE_PROTOCOL_REGISTRY: Mapping[ConditionId, LiveProtocolSpec] = MappingProxyType( + { + condition_id: LiveProtocolSpec( + condition_id=condition_id, + protocol_version=f"{condition_id}_live_v1", + prompt_version=LIVE_PROMPT_VERSIONS[condition_id], + executor=_EXECUTORS[condition_id], + ) + for condition_id in EVAL_CONDITION_IDS + } +) + + +async def execute_live_condition( + condition_id: ConditionId, + *, + task: PublicTask, + roster: Sequence[ProviderModelSpec], + cell_ceiling: int, + client: SequentialStageClient, +) -> str: + """Execute one frozen condition with sequential awaits and no Council fan-out.""" + + if len(roster) < ELITE_MIN_RESPONDERS: + raise ValueError("paid live rosters require at least three members") + caps = allocate_stage_caps( + condition_id, + roster_size=len(roster), + cell_ceiling=cell_ceiling, + ) + run = _Execution(condition_id, roster, caps, client) + return await LIVE_PROTOCOL_REGISTRY[condition_id].executor(run, _public_prompt(task)) diff --git a/src/conclave/evals/protocols.py b/src/conclave/evals/protocols.py index e4ca919..850e6df 100644 --- a/src/conclave/evals/protocols.py +++ b/src/conclave/evals/protocols.py @@ -6,6 +6,7 @@ import json import random from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING from .dataset import hash_public_tasks from .models import ( @@ -35,6 +36,17 @@ ) CONDITION_IDS: tuple[ConditionId, ...] = EVAL_CONDITION_IDS +if TYPE_CHECKING: + from .live_protocols import LiveProtocolSpec + + +def live_protocol_registry() -> Mapping[ConditionId, LiveProtocolSpec]: + """Return the versioned live executors' metadata without an import cycle.""" + + from .live_protocols import LIVE_PROTOCOL_REGISTRY + + return LIVE_PROTOCOL_REGISTRY + def condition_order(seed: int) -> tuple[ConditionId, ...]: """Return all frozen conditions in a deterministic seeded order.""" diff --git a/tests/evals/test_live_protocols.py b/tests/evals/test_live_protocols.py new file mode 100644 index 0000000..a27caef --- /dev/null +++ b/tests/evals/test_live_protocols.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Sequence + +import pytest + +from conclave import prompts +from conclave.evals.live_protocols import ( + LIVE_PROMPT_VERSIONS, + StageCall, + allocate_stage_caps, + execute_live_condition, + stage_call_sequence, +) +from conclave.evals.models import EVAL_CONDITION_IDS, ProviderModelSpec, PublicTask +from conclave.evals.protocols import live_protocol_registry +from conclave.models import ELITE_MIN_RESPONDERS, ModelAnswer + + +def _roster(size: int = 3) -> tuple[ProviderModelSpec, ...]: + return tuple( + ProviderModelSpec( + provider_id=f"fictional-provider-{index}", + model_id=f"fictional-model-{index}", + model_revision=f"fixture-r{index}", + ) + for index in range(1, size + 1) + ) + + +def _task() -> PublicTask: + return PublicTask( + task_id="public-decision", + prompt="Choose the strongest public option.", + reference_packets=("Public packet one.", "Public packet two."), + ) + + +class RecordingSequentialClient: + def __init__(self, *, fail_stages: Sequence[tuple[str, str]] = ()) -> None: + self.calls: list[StageCall] = [] + self.active = 0 + self.max_active = 0 + self._fail_stages = set(fail_stages) + + async def call(self, call: StageCall) -> ModelAnswer: + self.calls.append(call) + self.active += 1 + self.max_active = max(self.max_active, self.active) + await asyncio.sleep(0) + self.active -= 1 + if (call.stage, call.provider_id) in self._fail_stages: + return ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + error="bounded fake failure", + ) + return ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + answer=f"decision-artifact:{call.stage}:{len(self.calls)}", + ) + + +def _message_text(call: StageCall) -> str: + return "\n".join(message.content for message in call.messages) + + +def test_live_registry_covers_exactly_six_versioned_conditions() -> None: + registry = live_protocol_registry() + + assert tuple(registry) == EVAL_CONDITION_IDS + assert all(entry.protocol_version.endswith("_v1") for entry in registry.values()) + assert set(LIVE_PROMPT_VERSIONS) == set(EVAL_CONDITION_IDS) + assert all(LIVE_PROMPT_VERSIONS[condition] for condition in EVAL_CONDITION_IDS) + + +def test_stage_caps_are_positive_deterministic_and_sum_to_cell_ceiling() -> None: + for condition_id in EVAL_CONDITION_IDS: + call_count = len(stage_call_sequence(condition_id, roster_size=3)) + ceiling = call_count * 17 + 5 + + first = allocate_stage_caps(condition_id, roster_size=3, cell_ceiling=ceiling) + second = allocate_stage_caps(condition_id, roster_size=3, cell_ceiling=ceiling) + + assert first == second + assert len(first) == call_count + assert all(cap > 0 for cap in first) + assert sum(first) == ceiling + equal_share, remainder = divmod(ceiling, call_count) + assert first[-1] == equal_share + remainder + + +@pytest.mark.asyncio +async def test_single_and_self_refine_use_only_frozen_lead_member() -> None: + roster = _roster() + expected_stages = { + "single_frontier": ["initial"], + "self_refine": ["draft", "self_revision"], + } + + for condition_id, stages in expected_stages.items(): + client = RecordingSequentialClient() + output = await execute_live_condition( + condition_id, + task=_task(), + roster=roster, + cell_ceiling=120, + client=client, + ) + + assert [(call.stage, call.provider_id) for call in client.calls] == [ + (stage, roster[0].provider_id) for stage in stages + ] + assert output == f"decision-artifact:{stages[-1]}:{len(stages)}" + assert client.max_active == 1 + + +@pytest.mark.asyncio +async def test_multi_model_conditions_call_members_in_frozen_order() -> None: + roster = _roster() + providers = [member.provider_id for member in roster] + expected = { + "independent_synthesis": [ + *(("initial", provider) for provider in providers), + ("synthesis", providers[0]), + ], + "critique_only": [ + *(("initial", provider) for provider in providers), + *(("critique", provider) for provider in providers), + ("synthesis", providers[0]), + ], + "revision_only": [ + *(("initial", provider) for provider in providers), + *(("revision", provider) for provider in providers), + ("synthesis", providers[0]), + ], + "elite_full": [ + *(("initial", provider) for provider in providers), + *(("critique", provider) for provider in providers), + *(("revision", provider) for provider in providers), + ("synthesis", providers[0]), + ("verdict", providers[0]), + ], + } + + for condition_id, expected_calls in expected.items(): + client = RecordingSequentialClient() + output = await execute_live_condition( + condition_id, + task=_task(), + roster=roster, + cell_ceiling=1100, + client=client, + ) + + assert [(call.stage, call.provider_id) for call in client.calls] == expected_calls + assert output == f"decision-artifact:{expected_calls[-1][0]}:{len(expected_calls)}" + assert client.max_active == 1 + + +@pytest.mark.asyncio +async def test_critique_revision_and_elite_prompts_are_anonymized() -> None: + roster = _roster() + + for condition_id in ("critique_only", "revision_only", "elite_full"): + client = RecordingSequentialClient() + await execute_live_condition( + condition_id, + task=_task(), + roster=roster, + cell_ceiling=1100, + client=client, + ) + + peer_calls = [call for call in client.calls if call.stage != "initial"] + for call in peer_calls: + text = _message_text(call) + for member in roster: + assert member.provider_id not in text + assert member.model_id not in text + assert member.model_revision not in text + assert any("Model A" in _message_text(call) for call in peer_calls) + + +@pytest.mark.asyncio +async def test_elite_uses_current_versioned_prompt_builders_and_three_responder_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + original_critic = prompts.elite_critic_user + original_revision = prompts.elite_revision_user + builder_calls = {"critic": 0, "revision": 0} + + def tracked_critic(*args, **kwargs): + builder_calls["critic"] += 1 + return original_critic(*args, **kwargs) + + def tracked_revision(*args, **kwargs): + builder_calls["revision"] += 1 + return original_revision(*args, **kwargs) + + monkeypatch.setattr(prompts, "elite_critic_user", tracked_critic) + monkeypatch.setattr(prompts, "elite_revision_user", tracked_revision) + roster = _roster() + client = RecordingSequentialClient() + + await execute_live_condition( + "elite_full", + task=_task(), + roster=roster, + cell_ceiling=1100, + client=client, + ) + + assert ELITE_MIN_RESPONDERS == 3 + assert prompts.ELITE_PROMPT_VERSION in LIVE_PROMPT_VERSIONS["elite_full"] + assert builder_calls == {"critic": 1, "revision": 3} + + failed = RecordingSequentialClient(fail_stages=(("initial", roster[-1].provider_id),)) + with pytest.raises(ValueError, match="initial phase required 3 successful responders; got 2"): + await execute_live_condition( + "elite_full", + task=_task(), + roster=roster, + cell_ceiling=1100, + client=failed, + ) + assert [call.stage for call in failed.calls] == ["initial"] * 3 + + too_small_roster_client = RecordingSequentialClient() + with pytest.raises(ValueError, match="at least three"): + await execute_live_condition( + "elite_full", + task=_task(), + roster=_roster(2), + cell_ceiling=1100, + client=too_small_roster_client, + ) + assert too_small_roster_client.calls == [] + + +@pytest.mark.asyncio +async def test_too_small_cell_budget_fails_before_any_call() -> None: + roster = _roster() + + for condition_id in EVAL_CONDITION_IDS: + client = RecordingSequentialClient() + too_small = len(stage_call_sequence(condition_id, roster_size=len(roster))) - 1 + + with pytest.raises(ValueError, match="too small"): + await execute_live_condition( + condition_id, + task=_task(), + roster=roster, + cell_ceiling=too_small, + client=client, + ) + + assert client.calls == [] From e43bbac2e1495e025e14c4e8cb1be0d65a58940f Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 20:41:06 -0400 Subject: [PATCH 04/14] feat(evals): guard paid provider calls --- src/conclave/evals/__init__.py | 16 ++ src/conclave/evals/live.py | 329 +++++++++++++++++++++++++++++++ tests/evals/test_live_gateway.py | 252 +++++++++++++++++++++++ 3 files changed, 597 insertions(+) create mode 100644 src/conclave/evals/live.py create mode 100644 tests/evals/test_live_gateway.py diff --git a/src/conclave/evals/__init__.py b/src/conclave/evals/__init__.py index ad9260e..e721765 100644 --- a/src/conclave/evals/__init__.py +++ b/src/conclave/evals/__init__.py @@ -4,6 +4,15 @@ are versioned so recorded studies can be rejected when their contract changes. """ +from .live import ( + BudgetExceededError, + GatewayStoppedError, + LiveProviderClient, + PendingCall, + ProviderCallCostBasis, + ProviderCallReceipt, + ReservationBreachError, +) from .models import ( EVAL_SCHEMA_VERSION, ConditionSpec, @@ -29,6 +38,13 @@ "ScoreRecord", "StudyManifest", "StudyRun", + "BudgetExceededError", + "GatewayStoppedError", + "LiveProviderClient", + "PendingCall", + "ProviderCallCostBasis", + "ProviderCallReceipt", + "ReservationBreachError", "AdjudicationRecord", "GraderJudgment", "StudyScoreReport", diff --git a/src/conclave/evals/live.py b/src/conclave/evals/live.py new file mode 100644 index 0000000..e30d38b --- /dev/null +++ b/src/conclave/evals/live.py @@ -0,0 +1,329 @@ +"""Sequential paid-provider gateway with pessimistic budget reservations.""" + +from __future__ import annotations + +import asyncio +import inspect +from collections.abc import Awaitable, Callable +from decimal import ROUND_CEILING, Decimal, localcontext +from typing import Literal + +from pydantic import Field + +from conclave.models import ModelAnswer, TokenUsage +from conclave.providers import _receipt_error_category, call_model + +from .live_protocols import StageCall +from .models import EvalModel +from .pricing import ( + TOKENS_PER_MILLION, + USD_MICROCENT, + CallReservation, + ModelPrice, + PriceBook, + reserve_call_cost, +) + +ProviderCallOutcome = Literal["success", "failed"] +ProviderCallErrorCategory = Literal[ + "authentication", + "rate_limit", + "timeout", + "transport", + "provider_error", + "reservation_breach", +] +CostBasisSource = Literal["reported_usage", "full_reservation"] +_BOUNDED_PROVIDER_ERROR_CATEGORIES = frozenset( + {"authentication", "rate_limit", "timeout", "transport", "provider_error"} +) + + +class LiveGatewayError(RuntimeError): + """Base class for fail-closed live gateway errors.""" + + +class BudgetExceededError(LiveGatewayError): + """The next pessimistic reservation would cross the approved hard cap.""" + + +class ReservationBreachError(LiveGatewayError): + """A provider reported usage outside the pre-call reservation.""" + + +class GatewayStoppedError(LiveGatewayError): + """The gateway stopped after an accounting invariant was breached.""" + + +class ProviderCallCostBasis(EvalModel): + """Exact inputs used to reconcile one provider charge.""" + + source: CostBasisSource + prompt_tokens: int | None = Field(default=None, ge=0) + completion_tokens: int | None = Field(default=None, ge=0) + input_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) + output_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) + + +class PendingCall(EvalModel): + """Secret-free reservation persisted before any provider call begins.""" + + stage: str = Field(min_length=1) + provider_id: str = Field(min_length=1) + model_id: str = Field(min_length=1) + model_revision: str = Field(min_length=1) + max_output_tokens: int = Field(gt=0) + reservation: CallReservation + + +class ProviderCallReceipt(EvalModel): + """Bounded provider outcome and exact USD reconciliation facts.""" + + stage: str = Field(min_length=1) + provider_id: str = Field(min_length=1) + model_id: str = Field(min_length=1) + model_revision: str = Field(min_length=1) + max_output_tokens: int = Field(gt=0) + outcome: ProviderCallOutcome + error_category: ProviderCallErrorCategory | None = None + usage: TokenUsage | None = None + reserved_cost_usd: Decimal = Field(gt=0) + charged_cost_usd: Decimal = Field(ge=0) + cost_basis: ProviderCallCostBasis + + +CheckpointCallback = Callable[ + [PendingCall | None, tuple[ProviderCallReceipt, ...]], + Awaitable[None] | None, +] +CallModel = Callable[..., Awaitable[ModelAnswer]] + + +def _prompt_token_upper_bound(call: StageCall) -> int: + """Bound prompt content by UTF-8 bytes, conservatively above BPE tokens.""" + + return sum(len(message.content.encode("utf-8")) for message in call.messages) + + +def _provider_framing_token_allowance(call: StageCall) -> int: + """Reserve fixed per-request and per-message provider framing overhead.""" + + return 64 + (16 * len(call.messages)) + + +def _reported_usage_cost(price: ModelPrice, usage: TokenUsage) -> Decimal: + precision = max( + 64, + len(str(usage.prompt_tokens)) + + len(price.input_ceiling_usd_per_million_tokens.as_tuple().digits) + + 20, + len(str(usage.completion_tokens)) + + len(price.output_ceiling_usd_per_million_tokens.as_tuple().digits) + + 20, + ) + with localcontext() as context: + context.prec = precision + cost = ( + Decimal(usage.prompt_tokens) * price.input_ceiling_usd_per_million_tokens + + Decimal(usage.completion_tokens) * price.output_ceiling_usd_per_million_tokens + ) / TOKENS_PER_MILLION + return cost.quantize(USD_MICROCENT, rounding=ROUND_CEILING) + + +def _bounded_error_category(error: str) -> ProviderCallErrorCategory: + if error in _BOUNDED_PROVIDER_ERROR_CATEGORIES: + return error # type: ignore[return-value] + return _receipt_error_category(error) + + +class LiveProviderClient: + """Guard all eval provider calls with serialization and exact cost accounting.""" + + def __init__( + self, + *, + price_book: PriceBook, + hard_cap_usd: Decimal, + checkpoint: CheckpointCallback, + call_model_func: CallModel = call_model, + temperature: float = 0.0, + timeout: float = 120.0, + ) -> None: + if not isinstance(hard_cap_usd, Decimal): + raise TypeError("hard_cap_usd must be a Decimal") + if hard_cap_usd <= 0: + raise ValueError("hard_cap_usd must be positive") + self._price_book = price_book + self._hard_cap_usd = hard_cap_usd + self._checkpoint = checkpoint + self._call_model = call_model_func + self._temperature = temperature + self._timeout = timeout + self._lock = asyncio.Lock() + self._pending_call: PendingCall | None = None + self._receipts: list[ProviderCallReceipt] = [] + self._committed_cost_usd = Decimal("0") + self._stopped = False + + @property + def pending_call(self) -> PendingCall | None: + return self._pending_call + + @property + def receipts(self) -> tuple[ProviderCallReceipt, ...]: + return tuple(self._receipts) + + @property + def committed_cost_usd(self) -> Decimal: + return self._committed_cost_usd + + @property + def stopped(self) -> bool: + return self._stopped + + def _resolve_price(self, call: StageCall) -> ModelPrice: + identity = (call.provider_id, call.model_id, call.model_revision) + for price in self._price_book.entries: + if price.identity == identity: + return price + raise ValueError( + f"provider/model/revision is absent from the frozen price book: {'/'.join(identity)}" + ) + + async def _persist(self) -> None: + result = self._checkpoint(self._pending_call, self.receipts) + if inspect.isawaitable(result): + await result + + def _reserve(self, call: StageCall, price: ModelPrice) -> CallReservation: + return reserve_call_cost( + price, + prompt_token_upper_bound=_prompt_token_upper_bound(call), + prompt_template_token_allowance=0, + provider_framing_token_allowance=_provider_framing_token_allowance(call), + upstream_output_token_ceilings=call.upstream_output_token_ceilings, + max_output_tokens=call.max_output_tokens, + ) + + @staticmethod + def _usage_breaches(usage: TokenUsage, reservation: CallReservation) -> bool: + return ( + usage.prompt_tokens > reservation.input_token_upper_bound + or usage.completion_tokens > reservation.output_token_upper_bound + ) + + async def call(self, call: StageCall) -> ModelAnswer: + """Reserve, persist, execute, and reconcile one sequential stage call.""" + + async with self._lock: + if self._stopped: + raise GatewayStoppedError("live provider gateway is stopped") + + price = self._resolve_price(call) + reservation = self._reserve(call, price) + if self._committed_cost_usd + reservation.reserved_cost_usd > self._hard_cap_usd: + raise BudgetExceededError("provider call reservation would cross the hard cap") + + self._pending_call = PendingCall( + stage=call.stage, + provider_id=call.provider_id, + model_id=call.model_id, + model_revision=call.model_revision, + max_output_tokens=call.max_output_tokens, + reservation=reservation, + ) + await self._persist() + + messages = [ + {"role": message.role, "content": message.content} for message in call.messages + ] + try: + answer = await self._call_model( + call.provider_id, + call.model_id, + messages, + temperature=self._temperature, + timeout=self._timeout, + max_output_tokens=call.max_output_tokens, + ) + except Exception as exc: # noqa: BLE001 -- injected seam may raise + raw_error = f"{type(exc).__name__}: {exc}" + category = _bounded_error_category(raw_error) + answer = ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + error=category, + ) + + category: ProviderCallErrorCategory | None = None + if answer.error is not None: + category = _bounded_error_category(answer.error) + answer = answer.model_copy(update={"error": category}) + + usage = answer.usage + if usage is None: + charged_cost = reservation.reserved_cost_usd + cost_basis = ProviderCallCostBasis( + source="full_reservation", + input_ceiling_usd_per_million_tokens=( + reservation.input_ceiling_usd_per_million_tokens + ), + output_ceiling_usd_per_million_tokens=( + reservation.output_ceiling_usd_per_million_tokens + ), + ) + breached = False + else: + charged_cost = _reported_usage_cost(price, usage) + cost_basis = ProviderCallCostBasis( + source="reported_usage", + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + input_ceiling_usd_per_million_tokens=( + reservation.input_ceiling_usd_per_million_tokens + ), + output_ceiling_usd_per_million_tokens=( + reservation.output_ceiling_usd_per_million_tokens + ), + ) + breached = self._usage_breaches(usage, reservation) + + if breached: + category = "reservation_breach" + answer = answer.model_copy(update={"answer": None, "error": category}) + + receipt = ProviderCallReceipt( + stage=call.stage, + provider_id=call.provider_id, + model_id=call.model_id, + model_revision=call.model_revision, + max_output_tokens=call.max_output_tokens, + outcome="failed" if category is not None else "success", + error_category=category, + usage=usage, + reserved_cost_usd=reservation.reserved_cost_usd, + charged_cost_usd=charged_cost, + cost_basis=cost_basis, + ) + self._committed_cost_usd += charged_cost + self._receipts.append(receipt) + self._pending_call = None + if breached: + self._stopped = True + await self._persist() + + if breached: + raise ReservationBreachError("provider usage breached the call reservation") + return answer + + +__all__ = [ + "BudgetExceededError", + "GatewayStoppedError", + "LiveGatewayError", + "LiveProviderClient", + "PendingCall", + "ProviderCallCostBasis", + "ProviderCallReceipt", + "ReservationBreachError", +] diff --git a/tests/evals/test_live_gateway.py b/tests/evals/test_live_gateway.py new file mode 100644 index 0000000..1d7e040 --- /dev/null +++ b/tests/evals/test_live_gateway.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import asyncio +from decimal import Decimal + +import pytest +from pydantic import ValidationError + +from conclave.evals.live import ( + BudgetExceededError, + GatewayStoppedError, + LiveProviderClient, + ProviderCallReceipt, + ReservationBreachError, +) +from conclave.evals.live_protocols import ChatMessage, StageCall +from conclave.evals.pricing import ModelPrice, PriceBook +from conclave.models import ModelAnswer, TokenUsage + + +def _price_book( + *, + input_rate: str = "1000000", + output_rate: str = "1000000", +) -> PriceBook: + return PriceBook( + snapshot_id="fictional-gateway-prices", + captured_at="2026-07-18T12:00:00Z", + currency="USD", + entries=( + ModelPrice( + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + input_ceiling_usd_per_million_tokens=Decimal(input_rate), + output_ceiling_usd_per_million_tokens=Decimal(output_rate), + ), + ), + ) + + +def _stage_call(*, cap: int = 5, content: str = "x") -> StageCall: + return StageCall( + stage="initial", + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + messages=(ChatMessage(role="user", content=content),), + max_output_tokens=cap, + ) + + +@pytest.mark.asyncio +async def test_gateway_persists_reservation_before_calling_provider() -> None: + checkpoints: list[tuple[object | None, tuple[ProviderCallReceipt, ...]]] = [] + observed_caps: list[int | None] = [] + client: LiveProviderClient + + async def checkpoint(pending, receipts) -> None: + checkpoints.append((pending, receipts)) + + async def fake_call_model(name, model_id, messages, **kwargs): + assert client.pending_call is not None + assert checkpoints[-1][0] == client.pending_call + assert checkpoints[-1][1] == () + assert messages == [{"role": "user", "content": "x"}] + observed_caps.append(kwargs.get("max_output_tokens")) + return ModelAnswer( + name=name, + model_id=model_id, + answer="ok", + usage=TokenUsage(prompt_tokens=1, completion_tokens=2, total_tokens=3), + ) + + client = LiveProviderClient( + price_book=_price_book(), + hard_cap_usd=Decimal("100"), + call_model_func=fake_call_model, + checkpoint=checkpoint, + ) + + answer = await client.call(_stage_call(cap=5)) + + assert answer.ok + assert observed_caps == [5] + assert checkpoints[0][0] is not None + assert checkpoints[-1][0] is None + assert len(checkpoints[-1][1]) == 1 + + +@pytest.mark.asyncio +async def test_gateway_allows_only_one_in_flight_call() -> None: + active = 0 + maximum_active = 0 + + async def fake_call_model(name, model_id, messages, **kwargs): + nonlocal active, maximum_active + active += 1 + maximum_active = max(maximum_active, active) + await asyncio.sleep(0.01) + active -= 1 + return ModelAnswer( + name=name, + model_id=model_id, + answer="ok", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + client = LiveProviderClient( + price_book=_price_book(input_rate="1", output_rate="1"), + hard_cap_usd=Decimal("1"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + + answers = await asyncio.gather(client.call(_stage_call()), client.call(_stage_call())) + + assert all(answer.ok for answer in answers) + assert maximum_active == 1 + assert len(client.receipts) == 2 + + +@pytest.mark.asyncio +async def test_gateway_rejects_call_that_would_cross_hard_cap() -> None: + provider_calls = 0 + + async def fake_call_model(name, model_id, messages, **kwargs): + nonlocal provider_calls + provider_calls += 1 + return ModelAnswer(name=name, model_id=model_id, answer="must not run") + + client = LiveProviderClient( + price_book=_price_book(), + hard_cap_usd=Decimal("60"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + + with pytest.raises(BudgetExceededError, match="hard cap"): + await client.call(_stage_call(cap=5)) + + assert provider_calls == 0 + assert client.pending_call is None + assert client.receipts == () + assert client.committed_cost_usd == Decimal("0") + + +@pytest.mark.asyncio +async def test_gateway_prices_complete_usage_and_charges_reservation_when_missing() -> None: + answers = iter( + ( + ModelAnswer( + name="fictional-provider", + model_id="fictional/model", + answer="metered", + usage=TokenUsage(prompt_tokens=2, completion_tokens=3, total_tokens=5), + ), + ModelAnswer( + name="fictional-provider", + model_id="fictional/model", + answer="unmetered", + ), + ) + ) + + async def fake_call_model(name, model_id, messages, **kwargs): + return next(answers) + + client = LiveProviderClient( + price_book=_price_book(), + hard_cap_usd=Decimal("200"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + + await client.call(_stage_call(cap=5)) + await client.call(_stage_call(cap=5)) + + metered, unmetered = client.receipts + assert metered.cost_basis.source == "reported_usage" + assert metered.cost_basis.prompt_tokens == 2 + assert metered.cost_basis.completion_tokens == 3 + assert metered.charged_cost_usd == Decimal("5.000000") + assert unmetered.cost_basis.source == "full_reservation" + assert unmetered.cost_basis.prompt_tokens is None + assert unmetered.cost_basis.completion_tokens is None + assert unmetered.charged_cost_usd == unmetered.reserved_cost_usd + assert client.committed_cost_usd == (metered.charged_cost_usd + unmetered.charged_cost_usd) + + +@pytest.mark.asyncio +async def test_gateway_stops_on_reservation_breach() -> None: + provider_calls = 0 + + async def fake_call_model(name, model_id, messages, **kwargs): + nonlocal provider_calls + provider_calls += 1 + return ModelAnswer( + name=name, + model_id=model_id, + answer="provider exceeded cap", + usage=TokenUsage(prompt_tokens=1, completion_tokens=6, total_tokens=7), + ) + + client = LiveProviderClient( + price_book=_price_book(input_rate="1", output_rate="1"), + hard_cap_usd=Decimal("1"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + + with pytest.raises(ReservationBreachError, match="reservation"): + await client.call(_stage_call(cap=5)) + + assert client.stopped is True + assert client.receipts[-1].outcome == "failed" + assert client.receipts[-1].error_category == "reservation_breach" + with pytest.raises(GatewayStoppedError, match="stopped"): + await client.call(_stage_call(cap=5)) + assert provider_calls == 1 + + +@pytest.mark.asyncio +async def test_gateway_receipt_contains_bounded_error_category_not_raw_exception() -> None: + secret = "sk-live-secret-value" + raw_error = ( + f"401 from https://provider.invalid/private Authorization: Bearer {secret}; " + "x-api-key: raw-secret" + ) + + async def fake_call_model(name, model_id, messages, **kwargs): + raise RuntimeError(raw_error) + + client = LiveProviderClient( + price_book=_price_book(input_rate="1", output_rate="1"), + hard_cap_usd=Decimal("1"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + + answer = await client.call(_stage_call()) + receipt = client.receipts[-1] + payload = receipt.model_dump_json() + + assert answer.error == "authentication" + assert receipt.outcome == "failed" + assert receipt.error_category == "authentication" + assert receipt.cost_basis.source == "full_reservation" + for forbidden in (secret, raw_error, "https://", "Authorization", "Bearer", "x-api-key"): + assert forbidden not in payload + with pytest.raises(ValidationError): + ProviderCallReceipt.model_validate({**receipt.model_dump(), "raw_exception": raw_error}) From 6beb55bc1ee8382df7a0961da4537129158fbd10 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 20:49:57 -0400 Subject: [PATCH 05/14] feat(evals): checkpoint live study execution --- src/conclave/evals/live.py | 374 +++++++++++++++++++++++++++- tests/evals/test_live_checkpoint.py | 345 +++++++++++++++++++++++++ 2 files changed, 712 insertions(+), 7 deletions(-) create mode 100644 tests/evals/test_live_checkpoint.py diff --git a/src/conclave/evals/live.py b/src/conclave/evals/live.py index e30d38b..7e6821c 100644 --- a/src/conclave/evals/live.py +++ b/src/conclave/evals/live.py @@ -3,24 +3,32 @@ from __future__ import annotations import asyncio +import hashlib import inspect +import json +import os +import tempfile from collections.abc import Awaitable, Callable from decimal import ROUND_CEILING, Decimal, localcontext +from pathlib import Path from typing import Literal -from pydantic import Field +from pydantic import Field, ValidationError, model_validator +from conclave.adapters.base import redact from conclave.models import ModelAnswer, TokenUsage from conclave.providers import _receipt_error_category, call_model +from conclave.registry import PROVIDER_ENV_VARS from .live_protocols import StageCall -from .models import EvalModel +from .models import EVAL_SCHEMA_VERSION, EvalModel, RunRecord, Sha256Digest, StudyManifest from .pricing import ( TOKENS_PER_MILLION, USD_MICROCENT, CallReservation, ModelPrice, PriceBook, + hash_price_entries, reserve_call_cost, ) @@ -32,6 +40,7 @@ "transport", "provider_error", "reservation_breach", + "interrupted", ] CostBasisSource = Literal["reported_usage", "full_reservation"] _BOUNDED_PROVIDER_ERROR_CATEGORIES = frozenset( @@ -55,6 +64,14 @@ class GatewayStoppedError(LiveGatewayError): """The gateway stopped after an accounting invariant was breached.""" +class CheckpointValidationError(LiveGatewayError): + """A checkpoint is corrupt, tampered, or bound to different study inputs.""" + + +class CheckpointSecurityError(LiveGatewayError): + """A checkpoint payload failed the no-secret persistence boundary.""" + + class ProviderCallCostBasis(EvalModel): """Exact inputs used to reconcile one provider charge.""" @@ -92,6 +109,321 @@ class ProviderCallReceipt(EvalModel): cost_basis: ProviderCallCostBasis +class CheckpointBindings(EvalModel): + """Immutable inputs that make a checkpoint safe for exactly one live study.""" + + manifest_hash: Sha256Digest + price_book_hash: Sha256Digest + public_tasks_hash: Sha256Digest + hard_cap_usd: Decimal = Field(gt=0) + + +class ActiveCell(EvalModel): + """Cell state persisted while a planned run is not yet a final record.""" + + planned_run_id: str = Field(pattern=r"^run_[0-9a-f]{24}$") + receipt_start_index: int = Field(ge=0) + pending_call: PendingCall | None = None + + +def _canonical_value(value: object) -> object: + if isinstance(value, Decimal): + text = format(value, "f") + if "." in text: + text = text.rstrip("0").rstrip(".") + return "0" if text in {"-0", ""} else text + if isinstance(value, dict): + return {str(key): _canonical_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_canonical_value(item) for item in value] + return value + + +def _canonical_hash(payload: object) -> str: + canonical = json.dumps( + _canonical_value(payload), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" + + +class LiveCheckpoint(EvalModel): + """Secret-free, integrity-sealed state for one sequential live study.""" + + bindings: CheckpointBindings + committed_cost_usd: Decimal = Field(ge=0) + records: tuple[RunRecord, ...] = () + receipts: tuple[ProviderCallReceipt, ...] = () + active_cell: ActiveCell | None = None + checkpoint_hash: Sha256Digest + + @model_validator(mode="after") + def validate_integrity(self) -> LiveCheckpoint: + if self.checkpoint_hash != hash_live_checkpoint(self): + raise ValueError("checkpoint integrity hash mismatch") + record_ids = [record.planned_run_id for record in self.records] + if len(set(record_ids)) != len(record_ids): + raise ValueError("checkpoint records must have unique planned_run_id values") + if self.active_cell is not None: + if self.active_cell.planned_run_id in record_ids: + raise ValueError("active checkpoint cell must not already have a final record") + if self.active_cell.receipt_start_index > len(self.receipts): + raise ValueError("active checkpoint receipt_start_index exceeds receipt count") + receipt_cost = sum((receipt.charged_cost_usd for receipt in self.receipts), Decimal("0")) + if self.committed_cost_usd != receipt_cost: + raise ValueError("checkpoint committed cost must equal receipt charges") + if self.committed_cost_usd > self.bindings.hard_cap_usd: + raise ValueError("checkpoint committed cost exceeds the hard cap") + return self + + def should_execute(self, planned_run_id: str) -> bool: + """Return false for every completed or currently interrupted cell.""" + + if any(record.planned_run_id == planned_run_id for record in self.records): + return False + return self.active_cell is None or self.active_cell.planned_run_id != planned_run_id + + +def hash_study_manifest(manifest: StudyManifest) -> str: + """Return the canonical digest of the exact frozen live manifest.""" + + return _canonical_hash(manifest.model_dump(mode="python")) + + +def build_checkpoint_bindings( + manifest: StudyManifest, + price_book: PriceBook, + *, + hard_cap_usd: Decimal, +) -> CheckpointBindings: + """Bind checkpoint state to the exact manifest, prices, tasks, and ceiling.""" + + return CheckpointBindings( + manifest_hash=hash_study_manifest(manifest), + price_book_hash=hash_price_entries(price_book.entries), + public_tasks_hash=manifest.public_tasks_hash, + hard_cap_usd=hard_cap_usd, + ) + + +def hash_live_checkpoint(checkpoint: LiveCheckpoint) -> str: + """Return a canonical digest over checkpoint content and all study bindings.""" + + return _canonical_hash(checkpoint.model_dump(mode="python", exclude={"checkpoint_hash"})) + + +def create_live_checkpoint( + *, + bindings: CheckpointBindings, + records: tuple[RunRecord, ...] = (), + receipts: tuple[ProviderCallReceipt, ...] = (), + active_cell: ActiveCell | None = None, + committed_cost_usd: Decimal | None = None, +) -> LiveCheckpoint: + """Create and integrity-seal one immutable checkpoint snapshot.""" + + if committed_cost_usd is None: + committed_cost_usd = sum((receipt.charged_cost_usd for receipt in receipts), Decimal("0")) + unsealed = LiveCheckpoint.model_construct( + schema_version=EVAL_SCHEMA_VERSION, + bindings=bindings, + committed_cost_usd=committed_cost_usd, + records=tuple(records), + receipts=tuple(receipts), + active_cell=active_cell, + checkpoint_hash="sha256:" + ("0" * 64), + ) + payload = unsealed.model_dump(mode="python") + payload["checkpoint_hash"] = hash_live_checkpoint(unsealed) + return LiveCheckpoint.model_validate(payload) + + +def _active_provider_key_values() -> tuple[str, ...]: + values = [] + for names in PROVIDER_ENV_VARS.values(): + for name in names: + value = os.environ.get(name, "").strip() + if value: + values.append(value) + return tuple(dict.fromkeys(values)) + + +def _checkpoint_json(checkpoint: LiveCheckpoint) -> str: + try: + validated = LiveCheckpoint.model_validate(checkpoint.model_dump(mode="python")) + except (ValidationError, ValueError) as exc: + raise CheckpointValidationError("invalid live checkpoint state") from exc + return ( + json.dumps( + validated.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ) + + +def write_live_checkpoint(path: str | Path, checkpoint: LiveCheckpoint) -> None: + """Secret-scan and atomically replace a checkpoint in its destination directory.""" + + destination = Path(path) + payload = _checkpoint_json(checkpoint) + if redact(payload) != payload or any( + key_value in payload for key_value in _active_provider_key_values() + ): + raise CheckpointSecurityError("checkpoint payload contains secret material") + + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=destination.parent, + prefix=f".{destination.name}.", + suffix=".tmp", + delete=False, + ) as handle: + temporary_path = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, destination) + except BaseException: + if temporary_path is not None: + try: + temporary_path.unlink(missing_ok=True) + except OSError: + pass + raise + + +def load_live_checkpoint( + path: str | Path, + *, + expected_bindings: CheckpointBindings, +) -> LiveCheckpoint: + """Strictly load and bind an integrity-sealed checkpoint, failing closed.""" + + try: + with Path(path).open(encoding="utf-8") as handle: + payload = json.load(handle, parse_float=Decimal) + checkpoint = LiveCheckpoint.model_validate(payload) + except json.JSONDecodeError as exc: + raise CheckpointValidationError("invalid live checkpoint JSON") from exc + except (OSError, TypeError, ValidationError, ValueError) as exc: + label = "integrity" if "integrity" in str(exc).lower() else "invalid" + raise CheckpointValidationError(f"{label} live checkpoint") from exc + + for field in ( + "manifest_hash", + "price_book_hash", + "public_tasks_hash", + "hard_cap_usd", + ): + if getattr(checkpoint.bindings, field) != getattr(expected_bindings, field): + raise CheckpointValidationError(f"checkpoint {field} drift") + return checkpoint + + +def start_active_cell(checkpoint: LiveCheckpoint, *, planned_run_id: str) -> LiveCheckpoint: + """Persist the start of a cell before its first provider reservation.""" + + if checkpoint.active_cell is not None: + raise CheckpointValidationError("a checkpoint cell is already active") + if not checkpoint.should_execute(planned_run_id): + raise CheckpointValidationError("planned run already has a final record") + return create_live_checkpoint( + bindings=checkpoint.bindings, + records=checkpoint.records, + receipts=checkpoint.receipts, + active_cell=ActiveCell( + planned_run_id=planned_run_id, + receipt_start_index=len(checkpoint.receipts), + ), + committed_cost_usd=checkpoint.committed_cost_usd, + ) + + +def update_live_checkpoint( + checkpoint: LiveCheckpoint, + *, + pending_call: PendingCall | None, + receipts: tuple[ProviderCallReceipt, ...], +) -> LiveCheckpoint: + """Apply one gateway persistence transition without rewriting prior history.""" + + if checkpoint.active_cell is None: + raise CheckpointValidationError("gateway state requires an active checkpoint cell") + if ( + len(receipts) < len(checkpoint.receipts) + or tuple(receipts[: len(checkpoint.receipts)]) != checkpoint.receipts + ): + raise CheckpointValidationError("gateway receipts must preserve checkpoint history") + receipt_tuple = tuple(receipts) + committed = sum((receipt.charged_cost_usd for receipt in receipt_tuple), Decimal("0")) + return create_live_checkpoint( + bindings=checkpoint.bindings, + records=checkpoint.records, + receipts=receipt_tuple, + active_cell=checkpoint.active_cell.model_copy(update={"pending_call": pending_call}), + committed_cost_usd=committed, + ) + + +def _interrupted_receipt(pending: PendingCall) -> ProviderCallReceipt: + reservation = pending.reservation + return ProviderCallReceipt( + stage=pending.stage, + provider_id=pending.provider_id, + model_id=pending.model_id, + model_revision=pending.model_revision, + max_output_tokens=pending.max_output_tokens, + outcome="failed", + error_category="interrupted", + reserved_cost_usd=reservation.reserved_cost_usd, + charged_cost_usd=reservation.reserved_cost_usd, + cost_basis=ProviderCallCostBasis( + source="full_reservation", + input_ceiling_usd_per_million_tokens=(reservation.input_ceiling_usd_per_million_tokens), + output_ceiling_usd_per_million_tokens=( + reservation.output_ceiling_usd_per_million_tokens + ), + ), + ) + + +def recover_interrupted_checkpoint(checkpoint: LiveCheckpoint) -> LiveCheckpoint: + """Charge pending work and finalize the interrupted cell without retrying it.""" + + active = checkpoint.active_cell + if active is None: + return checkpoint + receipts = checkpoint.receipts + if active.pending_call is not None: + receipts = (*receipts, _interrupted_receipt(active.pending_call)) + cell_cost = sum( + (receipt.charged_cost_usd for receipt in receipts[active.receipt_start_index :]), + Decimal("0"), + ) + record = RunRecord( + planned_run_id=active.planned_run_id, + outcome="incomplete", + error_category="interrupted", + cost_usd=float(cell_cost), + cost_receipt_complete=True, + deviation_codes=("interrupted_cell_not_retried",), + ) + return create_live_checkpoint( + bindings=checkpoint.bindings, + records=(*checkpoint.records, record), + receipts=receipts, + committed_cost_usd=sum((receipt.charged_cost_usd for receipt in receipts), Decimal("0")), + ) + + CheckpointCallback = Callable[ [PendingCall | None, tuple[ProviderCallReceipt, ...]], Awaitable[None] | None, @@ -148,11 +480,19 @@ def __init__( call_model_func: CallModel = call_model, temperature: float = 0.0, timeout: float = 120.0, + resume_from: LiveCheckpoint | None = None, ) -> None: if not isinstance(hard_cap_usd, Decimal): raise TypeError("hard_cap_usd must be a Decimal") if hard_cap_usd <= 0: raise ValueError("hard_cap_usd must be positive") + if resume_from is not None: + if resume_from.active_cell is not None: + raise CheckpointValidationError( + "recover the active checkpoint cell before resuming the gateway" + ) + if resume_from.bindings.hard_cap_usd != hard_cap_usd: + raise CheckpointValidationError("checkpoint hard_cap_usd drift") self._price_book = price_book self._hard_cap_usd = hard_cap_usd self._checkpoint = checkpoint @@ -161,8 +501,10 @@ def __init__( self._timeout = timeout self._lock = asyncio.Lock() self._pending_call: PendingCall | None = None - self._receipts: list[ProviderCallReceipt] = [] - self._committed_cost_usd = Decimal("0") + self._receipts = list(resume_from.receipts) if resume_from is not None else [] + self._committed_cost_usd = ( + resume_from.committed_cost_usd if resume_from is not None else Decimal("0") + ) self._stopped = False @property @@ -191,9 +533,13 @@ def _resolve_price(self, call: StageCall) -> ModelPrice: ) async def _persist(self) -> None: - result = self._checkpoint(self._pending_call, self.receipts) - if inspect.isawaitable(result): - await result + try: + result = self._checkpoint(self._pending_call, self.receipts) + if inspect.isawaitable(result): + await result + except BaseException: + self._stopped = True + raise def _reserve(self, call: StageCall, price: ModelPrice) -> CallReservation: return reserve_call_cost( @@ -318,12 +664,26 @@ async def call(self, call: StageCall) -> ModelAnswer: __all__ = [ + "ActiveCell", "BudgetExceededError", + "CheckpointBindings", + "CheckpointSecurityError", + "CheckpointValidationError", "GatewayStoppedError", + "LiveCheckpoint", "LiveGatewayError", "LiveProviderClient", "PendingCall", "ProviderCallCostBasis", "ProviderCallReceipt", "ReservationBreachError", + "build_checkpoint_bindings", + "create_live_checkpoint", + "hash_live_checkpoint", + "hash_study_manifest", + "load_live_checkpoint", + "recover_interrupted_checkpoint", + "start_active_cell", + "update_live_checkpoint", + "write_live_checkpoint", ] diff --git a/tests/evals/test_live_checkpoint.py b/tests/evals/test_live_checkpoint.py new file mode 100644 index 0000000..35b50fe --- /dev/null +++ b/tests/evals/test_live_checkpoint.py @@ -0,0 +1,345 @@ +from __future__ import annotations + +import json +import os +from decimal import Decimal + +import pytest + +import conclave.evals.live as live_module +from conclave.adapters.base import redact as existing_redact +from conclave.evals.live import ( + ActiveCell, + CheckpointBindings, + CheckpointSecurityError, + CheckpointValidationError, + GatewayStoppedError, + LiveProviderClient, + PendingCall, + ProviderCallCostBasis, + ProviderCallReceipt, + create_live_checkpoint, + load_live_checkpoint, + recover_interrupted_checkpoint, + start_active_cell, + update_live_checkpoint, + write_live_checkpoint, +) +from conclave.evals.live_protocols import ChatMessage, StageCall +from conclave.evals.models import RunRecord +from conclave.evals.pricing import ModelPrice, PriceBook, reserve_call_cost +from conclave.models import ModelAnswer, TokenUsage + +DIGEST_A = "sha256:" + "a" * 64 +DIGEST_B = "sha256:" + "b" * 64 +DIGEST_C = "sha256:" + "c" * 64 +RUN_A = "run_" + "a" * 24 +RUN_B = "run_" + "b" * 24 + + +def _bindings(**updates: object) -> CheckpointBindings: + values: dict[str, object] = { + "manifest_hash": DIGEST_A, + "price_book_hash": DIGEST_B, + "public_tasks_hash": DIGEST_C, + "hard_cap_usd": Decimal("10.00"), + } + values.update(updates) + return CheckpointBindings(**values) + + +def _price_book() -> PriceBook: + return PriceBook( + snapshot_id="fictional-checkpoint-prices", + captured_at="2026-07-18T12:00:00Z", + currency="USD", + entries=( + ModelPrice( + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + input_ceiling_usd_per_million_tokens=Decimal("1"), + output_ceiling_usd_per_million_tokens=Decimal("1"), + ), + ), + ) + + +def _stage_call() -> StageCall: + return StageCall( + stage="initial", + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + messages=(ChatMessage(role="user", content="safe fixture prompt"),), + max_output_tokens=5, + ) + + +def _pending_call() -> PendingCall: + price = _price_book().entries[0] + reservation = reserve_call_cost( + price, + prompt_token_upper_bound=10, + prompt_template_token_allowance=0, + provider_framing_token_allowance=2, + upstream_output_token_ceilings=(), + max_output_tokens=5, + ) + return PendingCall( + stage="initial", + provider_id=price.provider_id, + model_id=price.model_id, + model_revision=price.model_revision, + max_output_tokens=5, + reservation=reservation, + ) + + +def _receipt(*, charged: str = "0.000002") -> ProviderCallReceipt: + return ProviderCallReceipt( + stage="initial", + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + max_output_tokens=5, + outcome="success", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + reserved_cost_usd=Decimal("0.000017"), + charged_cost_usd=Decimal(charged), + cost_basis=ProviderCallCostBasis( + source="reported_usage", + prompt_tokens=1, + completion_tokens=1, + input_ceiling_usd_per_million_tokens=Decimal("1"), + output_ceiling_usd_per_million_tokens=Decimal("1"), + ), + ) + + +@pytest.mark.asyncio +async def test_checkpoint_write_is_flush_fsync_replace_and_secret_scanned( + tmp_path, monkeypatch +) -> None: + destination = tmp_path / "nested" / "live-checkpoint.json" + destination.parent.mkdir() + checkpoint = create_live_checkpoint(bindings=_bindings()) + events: list[tuple[str, object]] = [] + redacted_payloads: list[str] = [] + real_fsync = os.fsync + real_replace = os.replace + + def tracking_redact(payload: str) -> str: + redacted_payloads.append(payload) + return existing_redact(payload) + + def tracking_fsync(fd: int) -> None: + assert os.fstat(fd).st_size > 0 + events.append(("fsync", fd)) + real_fsync(fd) + + def tracking_replace(source, target) -> None: + assert events and events[-1][0] == "fsync" + assert os.fspath(source).startswith(os.fspath(destination.parent)) + events.append(("replace", os.fspath(target))) + real_replace(source, target) + + monkeypatch.setattr(live_module, "redact", tracking_redact) + monkeypatch.setattr(live_module.os, "fsync", tracking_fsync) + monkeypatch.setattr(live_module.os, "replace", tracking_replace) + + write_live_checkpoint(destination, checkpoint) + + assert [event[0] for event in events] == ["fsync", "replace"] + assert redacted_payloads and existing_redact(redacted_payloads[-1]) == redacted_payloads[-1] + assert load_live_checkpoint(destination, expected_bindings=_bindings()) == checkpoint + + secret = "opaque-provider-key-fixture" + monkeypatch.setenv("OPENAI_API_KEY", secret) + unsafe = create_live_checkpoint( + bindings=_bindings(), + records=(RunRecord(planned_run_id=RUN_A, outcome="success", output=secret),), + ) + replacements_before = len([event for event in events if event[0] == "replace"]) + with pytest.raises(CheckpointSecurityError, match="secret"): + write_live_checkpoint(destination, unsafe) + assert len([event for event in events if event[0] == "replace"]) == replacements_before + assert load_live_checkpoint(destination, expected_bindings=_bindings()) == checkpoint + + provider_calls = 0 + persist_calls = 0 + + async def provider(name, model_id, messages, **kwargs): + nonlocal provider_calls + provider_calls += 1 + return ModelAnswer(name=name, model_id=model_id, answer="ok") + + def failing_persistence(pending, receipts) -> None: + nonlocal persist_calls + persist_calls += 1 + if persist_calls == 2: + raise OSError("fixture persistence failure") + + client = LiveProviderClient( + price_book=_price_book(), + hard_cap_usd=Decimal("10"), + checkpoint=failing_persistence, + call_model_func=provider, + ) + + with pytest.raises(OSError, match="persistence failure"): + await client.call(_stage_call()) + with pytest.raises(GatewayStoppedError, match="stopped"): + await client.call(_stage_call()) + assert provider_calls == 1 + + +def test_checkpoint_rejects_manifest_price_task_or_ceiling_drift(tmp_path) -> None: + path = tmp_path / "checkpoint.json" + checkpoint = create_live_checkpoint(bindings=_bindings()) + equivalent = create_live_checkpoint(bindings=_bindings(hard_cap_usd=Decimal("10.0"))) + assert equivalent.checkpoint_hash == checkpoint.checkpoint_hash + write_live_checkpoint(path, checkpoint) + + for field, changed in ( + ("manifest_hash", "sha256:" + "d" * 64), + ("price_book_hash", "sha256:" + "e" * 64), + ("public_tasks_hash", "sha256:" + "f" * 64), + ("hard_cap_usd", Decimal("9.99")), + ): + expected = _bindings().model_copy(update={field: changed}) + with pytest.raises(CheckpointValidationError, match=field): + load_live_checkpoint(path, expected_bindings=expected) + + +@pytest.mark.asyncio +async def test_resume_charges_pending_reservation_and_never_repeats_interrupted_cell( + tmp_path, +) -> None: + class SimulatedInterruption(BaseException): + pass + + path = tmp_path / "checkpoint.json" + state = start_active_cell(create_live_checkpoint(bindings=_bindings()), planned_run_id=RUN_A) + + def persist(pending, receipts) -> None: + nonlocal state + state = update_live_checkpoint(state, pending_call=pending, receipts=receipts) + write_live_checkpoint(path, state) + + seen_cells: list[str] = [] + current_cell = RUN_A + + async def interrupted_provider(name, model_id, messages, **kwargs): + seen_cells.append(current_cell) + raise SimulatedInterruption + + client = LiveProviderClient( + price_book=_price_book(), + hard_cap_usd=Decimal("10"), + checkpoint=persist, + call_model_func=interrupted_provider, + ) + with pytest.raises(SimulatedInterruption): + await client.call(_stage_call()) + + interrupted = load_live_checkpoint(path, expected_bindings=_bindings()) + assert interrupted.active_cell is not None + assert interrupted.active_cell.pending_call is not None + pending_cost = interrupted.active_cell.pending_call.reservation.reserved_cost_usd + + recovered = recover_interrupted_checkpoint(interrupted) + write_live_checkpoint(path, recovered) + resumed = load_live_checkpoint(path, expected_bindings=_bindings()) + + async def resumed_provider(name, model_id, messages, **kwargs): + seen_cells.append(current_cell) + return ModelAnswer( + name=name, + model_id=model_id, + answer="resumed", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + resumed_client = LiveProviderClient( + price_book=_price_book(), + hard_cap_usd=Decimal("10"), + checkpoint=lambda pending, receipts: None, + call_model_func=resumed_provider, + resume_from=resumed, + ) + for planned_run_id in (RUN_A, RUN_B): + if resumed.should_execute(planned_run_id): + current_cell = planned_run_id + await resumed_client.call(_stage_call()) + + assert seen_cells == [RUN_A, RUN_B] + interrupted_record = next( + record for record in resumed.records if record.planned_run_id == RUN_A + ) + assert interrupted_record.outcome == "incomplete" + assert interrupted_record.deviation_codes == ("interrupted_cell_not_retried",) + assert interrupted_record.cost_receipt_complete is True + assert Decimal(str(interrupted_record.cost_usd)) == pending_cost + assert resumed.committed_cost_usd == pending_cost + assert resumed.receipts[-1].charged_cost_usd == pending_cost + assert resumed.receipts[-1].cost_basis.source == "full_reservation" + + +def test_resume_preserves_completed_records_and_call_receipts() -> None: + completed_record = RunRecord( + planned_run_id=RUN_A, + outcome="success", + output="completed fixture output", + completion_tokens=1, + cost_usd=0.000002, + cost_receipt_complete=True, + ) + completed_receipt = _receipt() + pending = _pending_call() + checkpoint = create_live_checkpoint( + bindings=_bindings(), + records=(completed_record,), + receipts=(completed_receipt,), + active_cell=ActiveCell( + planned_run_id=RUN_B, + receipt_start_index=1, + pending_call=pending, + ), + committed_cost_usd=completed_receipt.charged_cost_usd, + ) + + recovered = recover_interrupted_checkpoint(checkpoint) + + assert recovered.records[0] == completed_record + assert recovered.receipts[0] == completed_receipt + assert recovered.records[1].planned_run_id == RUN_B + assert recovered.records[1].outcome == "incomplete" + assert recovered.receipts[1].charged_cost_usd == pending.reservation.reserved_cost_usd + assert recovered.committed_cost_usd == ( + completed_receipt.charged_cost_usd + pending.reservation.reserved_cost_usd + ) + + +def test_corrupt_or_tampered_checkpoint_fails_closed(tmp_path) -> None: + path = tmp_path / "checkpoint.json" + checkpoint = create_live_checkpoint(bindings=_bindings()) + write_live_checkpoint(path, checkpoint) + + path.write_text("{not-json", encoding="utf-8") + with pytest.raises(CheckpointValidationError, match="invalid"): + load_live_checkpoint(path, expected_bindings=_bindings()) + + write_live_checkpoint(path, checkpoint) + payload = json.loads(path.read_text(encoding="utf-8")) + payload["committed_cost_usd"] = "0.01" + path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(CheckpointValidationError, match="integrity"): + load_live_checkpoint(path, expected_bindings=_bindings()) + + write_live_checkpoint(path, checkpoint) + payload = json.loads(path.read_text(encoding="utf-8")) + payload["unexpected"] = True + path.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(CheckpointValidationError, match="invalid"): + load_live_checkpoint(path, expected_bindings=_bindings()) From a393eb5c31fa8fd5257e42b0eb6e6c680a971a3a Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 20:59:33 -0400 Subject: [PATCH 06/14] feat(evals): run capped live study cells --- src/conclave/evals/live.py | 27 ++ src/conclave/evals/runner.py | 344 ++++++++++++++++++++++++++ tests/evals/test_live_runner.py | 419 ++++++++++++++++++++++++++++++++ 3 files changed, 790 insertions(+) create mode 100644 tests/evals/test_live_runner.py diff --git a/src/conclave/evals/live.py b/src/conclave/evals/live.py index 7e6821c..5f9e22e 100644 --- a/src/conclave/evals/live.py +++ b/src/conclave/evals/live.py @@ -424,6 +424,22 @@ def recover_interrupted_checkpoint(checkpoint: LiveCheckpoint) -> LiveCheckpoint ) +def finish_active_cell(checkpoint: LiveCheckpoint, *, record: RunRecord) -> LiveCheckpoint: + """Finalize the active cell while preserving all receipt and cost history.""" + + active = checkpoint.active_cell + if active is None: + raise CheckpointValidationError("a final record requires an active checkpoint cell") + if active.planned_run_id != record.planned_run_id: + raise CheckpointValidationError("final record does not match the active checkpoint cell") + return create_live_checkpoint( + bindings=checkpoint.bindings, + records=(*checkpoint.records, record), + receipts=checkpoint.receipts, + committed_cost_usd=checkpoint.committed_cost_usd, + ) + + CheckpointCallback = Callable[ [PendingCall | None, tuple[ProviderCallReceipt, ...]], Awaitable[None] | None, @@ -637,6 +653,16 @@ async def call(self, call: StageCall) -> ModelAnswer: if breached: category = "reservation_breach" answer = answer.model_copy(update={"answer": None, "error": category}) + charged_cost = reservation.reserved_cost_usd + cost_basis = ProviderCallCostBasis( + source="full_reservation", + input_ceiling_usd_per_million_tokens=( + reservation.input_ceiling_usd_per_million_tokens + ), + output_ceiling_usd_per_million_tokens=( + reservation.output_ceiling_usd_per_million_tokens + ), + ) receipt = ProviderCallReceipt( stage=call.stage, @@ -679,6 +705,7 @@ async def call(self, call: StageCall) -> ModelAnswer: "ReservationBreachError", "build_checkpoint_bindings", "create_live_checkpoint", + "finish_active_cell", "hash_live_checkpoint", "hash_study_manifest", "load_live_checkpoint", diff --git a/src/conclave/evals/runner.py b/src/conclave/evals/runner.py index a5c4477..34f57de 100644 --- a/src/conclave/evals/runner.py +++ b/src/conclave/evals/runner.py @@ -6,8 +6,35 @@ import time from collections import Counter from collections.abc import Awaitable, Callable, Mapping, Sequence +from decimal import Decimal +from pathlib import Path + +from conclave.providers import call_model from .dataset import hash_public_tasks +from .live import ( + BudgetExceededError, + CallModel, + CheckpointValidationError, + GatewayStoppedError, + LiveCheckpoint, + LiveProviderClient, + ProviderCallReceipt, + ReservationBreachError, + build_checkpoint_bindings, + create_live_checkpoint, + finish_active_cell, + load_live_checkpoint, + recover_interrupted_checkpoint, + start_active_cell, + update_live_checkpoint, + write_live_checkpoint, +) +from .live_protocols import ( + LIVE_PROTOCOL_REGISTRY, + allocate_stage_caps, + execute_live_condition, +) from .models import ( EVAL_CONDITION_IDS, ConditionId, @@ -18,6 +45,9 @@ StudyManifest, StudyRun, ) +from .pricing import PriceBook, validate_price_book + +LIVE_HARD_CAP_USD = Decimal("10.00") ProtocolExecutor = Callable[ ..., @@ -183,3 +213,317 @@ async def run_study( total_cost_usd=sum(record.cost_usd for record in records), total_deviation_count=sum(len(record.deviation_codes) for record in records), ) + + +def _validate_live_inputs( + manifest: StudyManifest, + tasks: Sequence[PublicTask], + price_book: PriceBook, +) -> tuple[dict[str, PublicTask], dict[str, tuple]]: + design = manifest.frozen_design + if manifest.evidence_classification != "paid_exploratory_pilot" or design is None: + raise RunValidationError("live execution requires a paid exploratory frozen manifest") + if design.evidence_classification != "paid_exploratory_pilot": + raise RunValidationError("live execution accepts paid exploratory evidence only") + if Decimal(str(design.approved_spend_ceiling_usd)) != LIVE_HARD_CAP_USD: + raise RunValidationError("live execution requires an exact USD 10.00 frozen ceiling") + + task_by_id = {task.task_id: task for task in tasks} + if len(task_by_id) != len(tasks) or set(task_by_id) != set(manifest.task_ids): + raise RunValidationError("public tasks must exactly match the manifest task_ids") + if hash_public_tasks(tasks) != manifest.public_tasks_hash: + raise RunValidationError("public task content does not match the manifest hash") + + roster_by_id = {roster.roster_id: roster.members for roster in design.rosters} + if len(roster_by_id) != len(design.rosters): + raise RunValidationError("frozen live roster IDs must be unique") + if any(len(members) < 3 for members in roster_by_id.values()): + raise RunValidationError("paid live rosters require at least three members") + + expected_prompt_versions = { + condition: spec.prompt_version for condition, spec in LIVE_PROTOCOL_REGISTRY.items() + } + expected_protocol_versions = { + condition: spec.protocol_version for condition, spec in LIVE_PROTOCOL_REGISTRY.items() + } + if design.condition_prompt_versions != expected_prompt_versions: + raise RunValidationError("frozen live prompt versions do not match the runner") + if design.condition_protocol_versions != expected_protocol_versions: + raise RunValidationError("frozen live protocol versions do not match the runner") + + for planned_run in manifest.planned_runs: + members = roster_by_id.get(planned_run.roster_id) + if members is None: + raise RunValidationError("planned run references an unknown frozen roster") + try: + allocate_stage_caps( + planned_run.condition_id, + roster_size=len(members), + cell_ceiling=planned_run.max_output_tokens, + ) + except (TypeError, ValueError) as exc: + raise RunValidationError("planned run has an invalid live stage allocation") from exc + + try: + validate_price_book(price_book, frozen_design=design) + except ValueError as exc: + raise RunValidationError(str(exc)) from exc + return task_by_id, roster_by_id + + +def _load_or_create_live_checkpoint( + checkpoint_path: Path, + *, + manifest: StudyManifest, + price_book: PriceBook, +) -> LiveCheckpoint: + bindings = build_checkpoint_bindings( + manifest, + price_book, + hard_cap_usd=LIVE_HARD_CAP_USD, + ) + if checkpoint_path.exists(): + checkpoint = load_live_checkpoint(checkpoint_path, expected_bindings=bindings) + else: + checkpoint = create_live_checkpoint(bindings=bindings) + write_live_checkpoint(checkpoint_path, checkpoint) + + planned_ids = {planned_run.planned_run_id for planned_run in manifest.planned_runs} + record_ids = {record.planned_run_id for record in checkpoint.records} + active_id = checkpoint.active_cell.planned_run_id if checkpoint.active_cell else None + if not record_ids.issubset(planned_ids) or ( + active_id is not None and active_id not in planned_ids + ): + raise CheckpointValidationError("checkpoint contains an unplanned live cell") + if checkpoint.active_cell is not None: + checkpoint = recover_interrupted_checkpoint(checkpoint) + write_live_checkpoint(checkpoint_path, checkpoint) + return checkpoint + + +def _cell_receipts( + checkpoint: LiveCheckpoint, *, receipt_start_index: int +) -> tuple[ProviderCallReceipt, ...]: + return checkpoint.receipts[receipt_start_index:] + + +def _aggregate_live_execution( + receipts: Sequence[ProviderCallReceipt], + *, + latency_ms: float, + output: str | None = None, + error_category: str | None = None, + outcome: str | None = None, + deviation_codes: tuple[str, ...] = (), +) -> ProtocolExecution: + failed = tuple(receipt for receipt in receipts if receipt.outcome == "failed") + resolved_error = error_category or (failed[-1].error_category if failed else None) + if outcome is None: + outcome = "timed_out" if resolved_error == "timeout" else "failed" + if output is not None: + outcome = "success" + completion_tokens = ( + sum(receipt.usage.completion_tokens for receipt in receipts if receipt.usage is not None) + if receipts and not failed and all(receipt.usage is not None for receipt in receipts) + else None + ) + receipt_deviations = tuple( + dict.fromkeys( + f"provider_{receipt.error_category}" + for receipt in failed + if receipt.error_category is not None + ) + ) + cost = sum((receipt.charged_cost_usd for receipt in receipts), Decimal("0")) + return ProtocolExecution( + outcome=outcome, + output=output, + completion_tokens=completion_tokens, + latency_ms=latency_ms, + error_category=None if output is not None else resolved_error, + cost_usd=float(cost), + cost_receipt_complete=True, + deviation_codes=(*receipt_deviations, *deviation_codes), + ) + + +def _record_from_execution(planned_run_id: str, execution: ProtocolExecution) -> RunRecord: + return RunRecord( + planned_run_id=planned_run_id, + outcome=execution.outcome, + output=execution.output, + completion_tokens=execution.completion_tokens, + latency_ms=execution.latency_ms, + error_category=execution.error_category, + cost_usd=execution.cost_usd, + cost_receipt_complete=execution.cost_receipt_complete, + deviation_codes=execution.deviation_codes, + ) + + +def _materialize_remaining( + checkpoint: LiveCheckpoint, + planned_runs: Sequence[PlannedRun], + *, + error_category: str, +) -> LiveCheckpoint: + recorded = {record.planned_run_id for record in checkpoint.records} + additions = tuple( + RunRecord( + planned_run_id=planned_run.planned_run_id, + outcome="incomplete", + error_category=error_category, + cost_receipt_complete=True, + deviation_codes=(error_category,), + ) + for planned_run in planned_runs + if planned_run.planned_run_id not in recorded + ) + return create_live_checkpoint( + bindings=checkpoint.bindings, + records=(*checkpoint.records, *additions), + receipts=checkpoint.receipts, + committed_cost_usd=checkpoint.committed_cost_usd, + ) + + +def _build_live_study_run(manifest: StudyManifest, checkpoint: LiveCheckpoint) -> StudyRun: + by_id = {record.planned_run_id: record for record in checkpoint.records} + ordered = tuple(by_id[planned.planned_run_id] for planned in manifest.planned_runs) + records = validate_run_records(manifest, ordered) + return StudyRun( + study_id=manifest.study_id, + records=records, + total_planned_runs=len(manifest.planned_runs), + outcome_counts=dict(sorted(Counter(record.outcome for record in records).items())), + total_completion_tokens=sum(record.completion_tokens or 0 for record in records), + total_latency_ms=sum(record.latency_ms or 0.0 for record in records), + total_cost_usd=float(checkpoint.committed_cost_usd), + total_deviation_count=sum(len(record.deviation_codes) for record in records), + ) + + +async def run_live_study( + *, + manifest: StudyManifest, + tasks: Sequence[PublicTask], + price_book: PriceBook, + checkpoint_path: str | Path, + call_model_func: CallModel = call_model, +) -> StudyRun: + """Execute a frozen paid-exploratory matrix under the exact USD 10.00 cap.""" + + task_by_id, roster_by_id = _validate_live_inputs(manifest, tasks, price_book) + path = Path(checkpoint_path) + checkpoint = _load_or_create_live_checkpoint( + path, + manifest=manifest, + price_book=price_book, + ) + + def persist( + pending_call, + receipts: tuple[ProviderCallReceipt, ...], + ) -> None: + nonlocal checkpoint + checkpoint = update_live_checkpoint( + checkpoint, + pending_call=pending_call, + receipts=receipts, + ) + write_live_checkpoint(path, checkpoint) + + client = LiveProviderClient( + price_book=price_book, + hard_cap_usd=LIVE_HARD_CAP_USD, + checkpoint=persist, + call_model_func=call_model_func, + timeout=manifest.frozen_design.timeout_retry_policy.timeout_seconds, + resume_from=checkpoint, + ) + + planned_runs = manifest.planned_runs + for index, planned_run in enumerate(planned_runs): + if not checkpoint.should_execute(planned_run.planned_run_id): + continue + checkpoint = start_active_cell( + checkpoint, + planned_run_id=planned_run.planned_run_id, + ) + write_live_checkpoint(path, checkpoint) + receipt_start_index = checkpoint.active_cell.receipt_start_index + started = time.perf_counter() + try: + output = await execute_live_condition( + planned_run.condition_id, + task=task_by_id[planned_run.task_id], + roster=roster_by_id[planned_run.roster_id], + cell_ceiling=planned_run.max_output_tokens, + client=client, + ) + execution = _aggregate_live_execution( + _cell_receipts(checkpoint, receipt_start_index=receipt_start_index), + latency_ms=(time.perf_counter() - started) * 1000, + output=output, + ) + except BudgetExceededError: + execution = _aggregate_live_execution( + _cell_receipts(checkpoint, receipt_start_index=receipt_start_index), + latency_ms=(time.perf_counter() - started) * 1000, + error_category="budget_exhausted", + outcome="incomplete", + deviation_codes=("budget_exhausted",), + ) + checkpoint = finish_active_cell( + checkpoint, + record=_record_from_execution(planned_run.planned_run_id, execution), + ) + checkpoint = _materialize_remaining( + checkpoint, + planned_runs[index + 1 :], + error_category="budget_exhausted", + ) + write_live_checkpoint(path, checkpoint) + break + except (ReservationBreachError, GatewayStoppedError) as exc: + category = ( + "reservation_breach" + if isinstance(exc, ReservationBreachError) + else "gateway_stopped" + ) + execution = _aggregate_live_execution( + _cell_receipts(checkpoint, receipt_start_index=receipt_start_index), + latency_ms=(time.perf_counter() - started) * 1000, + error_category=category, + deviation_codes=(category,), + ) + checkpoint = finish_active_cell( + checkpoint, + record=_record_from_execution(planned_run.planned_run_id, execution), + ) + checkpoint = _materialize_remaining( + checkpoint, + planned_runs[index + 1 :], + error_category="gateway_stopped", + ) + write_live_checkpoint(path, checkpoint) + break + except Exception: # noqa: BLE001 -- bounded receipts determine the public failure + receipts = _cell_receipts(checkpoint, receipt_start_index=receipt_start_index) + execution = _aggregate_live_execution( + receipts, + latency_ms=(time.perf_counter() - started) * 1000, + error_category=( + None + if any(receipt.outcome == "failed" for receipt in receipts) + else "protocol_error" + ), + ) + + checkpoint = finish_active_cell( + checkpoint, + record=_record_from_execution(planned_run.planned_run_id, execution), + ) + write_live_checkpoint(path, checkpoint) + + return _build_live_study_run(manifest, checkpoint) diff --git a/tests/evals/test_live_runner.py b/tests/evals/test_live_runner.py new file mode 100644 index 0000000..956ad65 --- /dev/null +++ b/tests/evals/test_live_runner.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import json +from collections import Counter +from decimal import Decimal + +import pytest + +import conclave.evals.dataset as dataset_module +from conclave.evals.live import ( + CheckpointValidationError, + build_checkpoint_bindings, + create_live_checkpoint, + load_live_checkpoint, + write_live_checkpoint, +) +from conclave.evals.live_protocols import LIVE_PROTOCOL_REGISTRY, stage_call_sequence +from conclave.evals.models import ( + AnalysisGateConfig, + BootstrapConfig, + ExclusionDeviationPolicy, + FrozenStudyDesign, + PriceSnapshot, + ProviderModelSpec, + PublicTask, + RandomizationConfig, + RosterSpec, + TimeoutRetryPolicy, +) +from conclave.evals.pricing import ModelPrice, PriceBook, hash_price_entries +from conclave.evals.protocols import CONDITION_IDS, build_study_manifest +from conclave.evals.runner import RunValidationError, run_live_study, validate_run_records +from conclave.models import ModelAnswer, TokenUsage + +DIGEST = "sha256:" + "a" * 64 +HARD_CAP = Decimal("10.00") + + +def _rosters(*, members_per_roster: int = 3) -> tuple[RosterSpec, ...]: + rosters = [] + member_number = 0 + for roster_name in ("a", "b"): + members = [] + for _ in range(members_per_roster): + member_number += 1 + members.append( + ProviderModelSpec( + provider_id=f"fictional-provider-{member_number}", + model_id=f"fictional/model-{member_number}", + model_revision=f"fixture-r{member_number}", + ) + ) + rosters.append(RosterSpec(roster_id=f"fictional-roster-{roster_name}", members=members)) + return tuple(rosters) + + +def _live_inputs( + *, + rate: str = "1", + members_per_roster: int = 3, + evidence_classification: str = "paid_exploratory_pilot", + approved_spend_ceiling_usd: float = 10.0, + version_drift: bool = False, +): + tasks = [ + PublicTask( + task_id="fictional-task", + prompt="Choose the safest fictional option.", + reference_packets=("All facts in this packet are fictional.",), + ) + ] + rosters = _rosters(members_per_roster=members_per_roster) + entries = tuple( + ModelPrice( + provider_id=member.provider_id, + model_id=member.model_id, + model_revision=member.model_revision, + input_ceiling_usd_per_million_tokens=Decimal(rate), + output_ceiling_usd_per_million_tokens=Decimal(rate), + ) + for roster in rosters + for member in roster.members + ) + price_book = PriceBook( + snapshot_id="fictional-live-runner-prices", + captured_at="2026-07-18T12:00:00Z", + currency="USD", + entries=entries, + ) + prompt_versions = { + condition: spec.prompt_version for condition, spec in LIVE_PROTOCOL_REGISTRY.items() + } + if version_drift: + prompt_versions["single_frontier"] = "drifted-prompt-version" + design = FrozenStudyDesign( + evidence_classification=evidence_classification, + base_commit="1" * 40, + task_family_map={"fictional-task": "fixture-family"}, + rosters=rosters, + condition_prompt_versions=prompt_versions, + condition_protocol_versions={ + condition: spec.protocol_version for condition, spec in LIVE_PROTOCOL_REGISTRY.items() + }, + generation_settings_hash=DIGEST, + evaluator_version="fictional-evaluator-v1", + analysis_code_hash=DIGEST, + rubric_hash=DIGEST, + grader_instructions_hash=DIGEST, + grader_keys_hash=DIGEST, + exclusion_deviation_policy=ExclusionDeviationPolicy(), + timeout_retry_policy=TimeoutRetryPolicy(timeout_seconds=1, retry_attempts=0), + randomization=RandomizationConfig(master_seed=20260718), + bootstrap=BootstrapConfig(seed=20260718, samples=10), + analysis_gates=AnalysisGateConfig( + primary_baseline="single_frontier", + absolute_p95_latency_seconds=60, + minimum_confirmatory_tasks=2, + ), + price_snapshot=PriceSnapshot( + snapshot_id=price_book.snapshot_id, + captured_at=price_book.captured_at, + currency=price_book.currency, + prices_hash=hash_price_entries(price_book.entries), + ), + approved_spend_ceiling_usd=approved_spend_ceiling_usd, + preregistration_id=( + "fixture:confirmatory" if evidence_classification == "confirmatory" else None + ), + preregistration_hash=(DIGEST if evidence_classification == "confirmatory" else None), + ) + manifest = build_study_manifest( + study_id="fictional-paid-pilot", + tasks=tasks, + replicates=1, + seed=20260718, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 64), + frozen_design=design, + ) + return tasks, manifest, price_book + + +async def _successful_provider(name, model_id, messages, **kwargs): + del messages, kwargs + return ModelAnswer( + name=name, + model_id=model_id, + answer=f"fictional decision from {name}", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + +@pytest.mark.asyncio +async def test_live_runner_requires_paid_exploratory_frozen_manifest(tmp_path) -> None: + provider_calls = 0 + + async def forbidden_provider(*args, **kwargs): + nonlocal provider_calls + del args, kwargs + provider_calls += 1 + raise AssertionError("validation must finish before provider configuration or keys") + + invalid_cases = [] + legacy_tasks = [PublicTask(task_id="legacy-task", prompt="Legacy")] + legacy_manifest = build_study_manifest( + study_id="legacy", + tasks=legacy_tasks, + replicates=1, + seed=1, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 64), + ) + _, _, valid_book = _live_inputs() + invalid_cases.append((legacy_tasks, legacy_manifest, valid_book)) + invalid_cases.append(_live_inputs(evidence_classification="confirmatory")) + invalid_cases.append(_live_inputs(approved_spend_ceiling_usd=9.99)) + invalid_cases.append(_live_inputs(members_per_roster=2)) + invalid_cases.append(_live_inputs(version_drift=True)) + + for index, (tasks, manifest, price_book) in enumerate(invalid_cases): + with pytest.raises(RunValidationError): + await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=tmp_path / f"invalid-{index}.json", + call_model_func=forbidden_provider, + ) + + tasks, manifest, price_book = _live_inputs() + drifted_book = price_book.model_copy(update={"snapshot_id": "drifted-snapshot"}) + with pytest.raises(RunValidationError, match="snapshot"): + await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=drifted_book, + checkpoint_path=tmp_path / "snapshot-drift.json", + call_model_func=forbidden_provider, + ) + + changed_tasks = [tasks[0].model_copy(update={"prompt": "Changed after freezing."})] + with pytest.raises(RunValidationError, match="task"): + await run_live_study( + manifest=manifest, + tasks=changed_tasks, + price_book=price_book, + checkpoint_path=tmp_path / "task-drift.json", + call_model_func=forbidden_provider, + ) + + checkpoint_path = tmp_path / "checkpoint-drift.json" + bindings = build_checkpoint_bindings(manifest, price_book, hard_cap_usd=HARD_CAP) + drifted_bindings = bindings.model_copy(update={"manifest_hash": DIGEST}) + write_live_checkpoint( + checkpoint_path, + create_live_checkpoint(bindings=drifted_bindings), + ) + with pytest.raises(CheckpointValidationError, match="manifest_hash"): + await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=checkpoint_path, + call_model_func=forbidden_provider, + ) + + assert provider_calls == 0 + + +@pytest.mark.asyncio +async def test_live_runner_executes_manifest_order_and_builds_complete_study_run( + tmp_path, +) -> None: + tasks, manifest, price_book = _live_inputs() + checkpoint_path = tmp_path / "live-checkpoint.json" + calls: list[tuple[str, str, str, int]] = [] + + async def recording_provider(name, model_id, messages, **kwargs): + del messages + payload = json.loads(checkpoint_path.read_text(encoding="utf-8")) + active = payload["active_cell"] + assert active["pending_call"] is not None + calls.append((active["planned_run_id"], name, model_id, kwargs["max_output_tokens"])) + return await _successful_provider(name, model_id, (), **kwargs) + + study_run = await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=checkpoint_path, + call_model_func=recording_provider, + ) + + expected_ids = tuple(run.planned_run_id for run in manifest.planned_runs) + first_seen_ids = tuple(dict.fromkeys(planned_run_id for planned_run_id, *_ in calls)) + call_counts = Counter(planned_run_id for planned_run_id, *_ in calls) + + assert first_seen_ids == expected_ids + assert call_counts == { + run.planned_run_id: len(stage_call_sequence(run.condition_id, roster_size=3)) + for run in manifest.planned_runs + } + assert tuple(record.planned_run_id for record in study_run.records) == expected_ids + assert study_run.outcome_counts == {"success": len(manifest.planned_runs)} + assert validate_run_records(manifest, study_run.records) == study_run.records + assert study_run.total_cost_usd <= 10.0 + assert ( + load_live_checkpoint( + checkpoint_path, + expected_bindings=build_checkpoint_bindings( + manifest, price_book, hard_cap_usd=HARD_CAP + ), + ).active_cell + is None + ) + + +@pytest.mark.asyncio +async def test_budget_exhaustion_makes_no_call_and_marks_all_remaining_cells_incomplete( + tmp_path, +) -> None: + tasks, manifest, price_book = _live_inputs(rate="100000000") + provider_calls = 0 + + async def forbidden_provider(*args, **kwargs): + nonlocal provider_calls + del args, kwargs + provider_calls += 1 + raise AssertionError("a rejected reservation must not call the provider") + + study_run = await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=tmp_path / "budget-checkpoint.json", + call_model_func=forbidden_provider, + ) + + assert provider_calls == 0 + assert len(study_run.records) == len(manifest.planned_runs) + assert all(record.outcome == "incomplete" for record in study_run.records) + assert all(record.error_category == "budget_exhausted" for record in study_run.records) + assert all(record.deviation_codes == ("budget_exhausted",) for record in study_run.records) + assert study_run.total_cost_usd == 0.0 + assert validate_run_records(manifest, study_run.records) == study_run.records + + +@pytest.mark.parametrize( + "failure_mode", + ("timeout", "missing_usage", "provider_error", "reservation_breach"), +) +@pytest.mark.asyncio +async def test_timeout_missing_usage_and_provider_error_never_exceed_ten_dollars( + tmp_path, + failure_mode, +) -> None: + tasks, manifest, price_book = _live_inputs(rate="10") + + async def provider(name, model_id, messages, **kwargs): + del messages + if failure_mode == "timeout": + raise TimeoutError("fictional provider timed out") + if failure_mode == "missing_usage": + return ModelAnswer(name=name, model_id=model_id, answer="unmetered fixture answer") + if failure_mode == "provider_error": + return ModelAnswer(name=name, model_id=model_id, error="fictional provider error") + return ModelAnswer( + name=name, + model_id=model_id, + answer="over reservation fixture answer", + usage=TokenUsage( + prompt_tokens=1, + completion_tokens=kwargs["max_output_tokens"] + 1, + total_tokens=kwargs["max_output_tokens"] + 2, + ), + ) + + checkpoint_path = tmp_path / f"{failure_mode}.json" + study_run = await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=checkpoint_path, + call_model_func=provider, + ) + checkpoint = load_live_checkpoint( + checkpoint_path, + expected_bindings=build_checkpoint_bindings(manifest, price_book, hard_cap_usd=HARD_CAP), + ) + + assert study_run.total_cost_usd <= 10.0 + assert checkpoint.committed_cost_usd <= HARD_CAP + assert sum((receipt.charged_cost_usd for receipt in checkpoint.receipts), Decimal()) <= HARD_CAP + assert validate_run_records(manifest, study_run.records) == study_run.records + assert "fictional provider timed out" not in checkpoint.model_dump_json() + if failure_mode == "timeout": + assert any(record.outcome == "timed_out" for record in study_run.records) + assert all(record.error_category == "timeout" for record in study_run.records) + elif failure_mode == "missing_usage": + assert all(record.outcome == "success" for record in study_run.records) + elif failure_mode == "provider_error": + assert any(record.outcome == "failed" for record in study_run.records) + assert all(record.error_category == "provider_error" for record in study_run.records) + else: + assert study_run.records[0].error_category == "reservation_breach" + assert all(record.outcome == "incomplete" for record in study_run.records[1:]) + + +@pytest.mark.asyncio +async def test_final_records_cover_every_planned_run_exactly_once(tmp_path) -> None: + tasks, manifest, price_book = _live_inputs(rate="10") + calls = 0 + + async def provider(name, model_id, messages, **kwargs): + nonlocal calls + del messages, kwargs + calls += 1 + if calls == 4: + return ModelAnswer(name=name, model_id=model_id, error="bounded fixture failure") + return ModelAnswer( + name=name, + model_id=model_id, + answer=f"fixture answer {calls}", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + study_run = await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=tmp_path / "coverage.json", + call_model_func=provider, + ) + + expected = Counter(run.planned_run_id for run in manifest.planned_runs) + actual = Counter(record.planned_run_id for record in study_run.records) + assert actual == expected + assert all(count == 1 for count in actual.values()) + assert validate_run_records(manifest, study_run.records) == study_run.records + + +@pytest.mark.asyncio +async def test_runner_never_loads_grader_keys(tmp_path, monkeypatch) -> None: + tasks, manifest, price_book = _live_inputs() + + def forbidden_grader_key_load(*args, **kwargs): + del args, kwargs + raise AssertionError("the live execution boundary must never load grader keys") + + monkeypatch.setattr(dataset_module, "load_grader_keys", forbidden_grader_key_load) + + study_run = await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=tmp_path / "no-grader-keys.json", + call_model_func=_successful_provider, + ) + + assert len(study_run.records) == len(manifest.planned_runs) + assert validate_run_records(manifest, study_run.records) == study_run.records From f93f6d3e667c764f032a20e66844747ec51df7ab Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 21:07:23 -0400 Subject: [PATCH 07/14] feat(evals): estimate capped live studies --- src/conclave/evals/live.py | 144 ++++++++++++++++++++++++++---- tests/evals/test_live_estimate.py | 136 ++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+), 18 deletions(-) create mode 100644 tests/evals/test_live_estimate.py diff --git a/src/conclave/evals/live.py b/src/conclave/evals/live.py index 5f9e22e..29db457 100644 --- a/src/conclave/evals/live.py +++ b/src/conclave/evals/live.py @@ -8,7 +8,7 @@ import json import os import tempfile -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Sequence from decimal import ROUND_CEILING, Decimal, localcontext from pathlib import Path from typing import Literal @@ -20,8 +20,16 @@ from conclave.providers import _receipt_error_category, call_model from conclave.registry import PROVIDER_ENV_VARS -from .live_protocols import StageCall -from .models import EVAL_SCHEMA_VERSION, EvalModel, RunRecord, Sha256Digest, StudyManifest +from .live_protocols import LIVE_PROTOCOL_REGISTRY, StageCall, execute_live_condition +from .models import ( + EVAL_SCHEMA_VERSION, + ConditionId, + EvalModel, + PublicTask, + RunRecord, + Sha256Digest, + StudyManifest, +) from .pricing import ( TOKENS_PER_MILLION, USD_MICROCENT, @@ -109,6 +117,20 @@ class ProviderCallReceipt(EvalModel): cost_basis: ProviderCallCostBasis +class LiveStudyEstimate(EvalModel): + """Network-free pessimistic cost summary for one frozen live study.""" + + planned_cells: int = Field(ge=1) + maximum_call_count: int = Field(ge=1) + per_roster_upper_bound_usd: dict[str, Decimal] + per_condition_upper_bound_usd: dict[ConditionId, Decimal] + largest_reservation_usd: Decimal = Field(gt=0) + total_upper_bound_usd: Decimal = Field(gt=0) + ceiling_usd: Decimal = Field(gt=0) + headroom_usd: Decimal + fits_ceiling: bool + + class CheckpointBindings(EvalModel): """Immutable inputs that make a checkpoint safe for exactly one live study.""" @@ -459,6 +481,27 @@ def _provider_framing_token_allowance(call: StageCall) -> int: return 64 + (16 * len(call.messages)) +def _resolve_call_price(price_book: PriceBook, call: StageCall) -> ModelPrice: + identity = (call.provider_id, call.model_id, call.model_revision) + for price in price_book.entries: + if price.identity == identity: + return price + raise ValueError( + f"provider/model/revision is absent from the frozen price book: {'/'.join(identity)}" + ) + + +def _reserve_stage_call(call: StageCall, price: ModelPrice) -> CallReservation: + return reserve_call_cost( + price, + prompt_token_upper_bound=_prompt_token_upper_bound(call), + prompt_template_token_allowance=0, + provider_framing_token_allowance=_provider_framing_token_allowance(call), + upstream_output_token_ceilings=call.upstream_output_token_ceilings, + max_output_tokens=call.max_output_tokens, + ) + + def _reported_usage_cost(price: ModelPrice, usage: TokenUsage) -> Decimal: precision = max( 64, @@ -540,13 +583,7 @@ def stopped(self) -> bool: return self._stopped def _resolve_price(self, call: StageCall) -> ModelPrice: - identity = (call.provider_id, call.model_id, call.model_revision) - for price in self._price_book.entries: - if price.identity == identity: - return price - raise ValueError( - f"provider/model/revision is absent from the frozen price book: {'/'.join(identity)}" - ) + return _resolve_call_price(self._price_book, call) async def _persist(self) -> None: try: @@ -558,14 +595,7 @@ async def _persist(self) -> None: raise def _reserve(self, call: StageCall, price: ModelPrice) -> CallReservation: - return reserve_call_cost( - price, - prompt_token_upper_bound=_prompt_token_upper_bound(call), - prompt_template_token_allowance=0, - provider_framing_token_allowance=_provider_framing_token_allowance(call), - upstream_output_token_ceilings=call.upstream_output_token_ceilings, - max_output_tokens=call.max_output_tokens, - ) + return _reserve_stage_call(call, price) @staticmethod def _usage_breaches(usage: TokenUsage, reservation: CallReservation) -> bool: @@ -689,6 +719,82 @@ async def call(self, call: StageCall) -> ModelAnswer: return answer +class _LiveEstimateClient: + """Walk live protocols while recording reservations without provider access.""" + + def __init__(self, price_book: PriceBook) -> None: + self._price_book = price_book + self.reservations: list[CallReservation] = [] + + async def call(self, call: StageCall) -> ModelAnswer: + price = _resolve_call_price(self._price_book, call) + self.reservations.append(_reserve_stage_call(call, price)) + return ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + answer="x" * call.max_output_tokens, + ) + + +async def estimate_live_study( + *, + manifest: StudyManifest, + tasks: Sequence[PublicTask], + price_book: PriceBook, +) -> LiveStudyEstimate: + """Estimate the full frozen live matrix without loading config, keys, or transport.""" + + from .runner import LIVE_HARD_CAP_USD, _validate_live_inputs + + task_by_id, roster_by_id = _validate_live_inputs(manifest, tasks, price_book) + design = manifest.frozen_design + if design is None: # pragma: no cover - enforced by shared validation + raise ValueError("live estimate requires a frozen study design") + + client = _LiveEstimateClient(price_book) + per_roster = {roster.roster_id: Decimal("0") for roster in design.rosters} + per_condition = {condition: Decimal("0") for condition in LIVE_PROTOCOL_REGISTRY} + + for planned_run in manifest.planned_runs: + reservation_start = len(client.reservations) + await execute_live_condition( + planned_run.condition_id, + task=task_by_id[planned_run.task_id], + roster=roster_by_id[planned_run.roster_id], + cell_ceiling=planned_run.max_output_tokens, + client=client, + ) + cell_upper_bound = sum( + ( + reservation.reserved_cost_usd + for reservation in client.reservations[reservation_start:] + ), + Decimal("0"), + ) + per_roster[planned_run.roster_id] += cell_upper_bound + per_condition[planned_run.condition_id] += cell_upper_bound + + reservation_costs = tuple(reservation.reserved_cost_usd for reservation in client.reservations) + total_upper_bound = sum(reservation_costs, Decimal("0")) + ceiling = Decimal(str(design.approved_spend_ceiling_usd)) + estimate = LiveStudyEstimate( + planned_cells=len(manifest.planned_runs), + maximum_call_count=len(reservation_costs), + per_roster_upper_bound_usd=per_roster, + per_condition_upper_bound_usd=per_condition, + largest_reservation_usd=max(reservation_costs), + total_upper_bound_usd=total_upper_bound, + ceiling_usd=ceiling, + headroom_usd=ceiling - total_upper_bound, + fits_ceiling=total_upper_bound <= ceiling, + ) + if not estimate.fits_ceiling: + raise BudgetExceededError("live study worst-case estimate exceeds the frozen ceiling") + if estimate.ceiling_usd != LIVE_HARD_CAP_USD: # pragma: no cover - shared validation + raise ValueError("live estimate ceiling drift") + return estimate + + __all__ = [ "ActiveCell", "BudgetExceededError", @@ -699,12 +805,14 @@ async def call(self, call: StageCall) -> ModelAnswer: "LiveCheckpoint", "LiveGatewayError", "LiveProviderClient", + "LiveStudyEstimate", "PendingCall", "ProviderCallCostBasis", "ProviderCallReceipt", "ReservationBreachError", "build_checkpoint_bindings", "create_live_checkpoint", + "estimate_live_study", "finish_active_cell", "hash_live_checkpoint", "hash_study_manifest", diff --git a/tests/evals/test_live_estimate.py b/tests/evals/test_live_estimate.py new file mode 100644 index 0000000..a1b9a4f --- /dev/null +++ b/tests/evals/test_live_estimate.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from decimal import Decimal + +import pytest +from pydantic import ValidationError + +import conclave.config as config_module +import conclave.evals.live as live_module +import conclave.registry as registry_module +import conclave.transport as transport_module +from conclave.evals.live import BudgetExceededError, estimate_live_study +from conclave.evals.live_protocols import allocate_stage_caps, stage_call_sequence +from tests.evals.test_live_runner import _live_inputs + + +@pytest.mark.asyncio +async def test_dry_run_walks_same_calls_without_loading_keys_or_transport( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tasks, manifest, price_book = _live_inputs() + + def forbidden(*args, **kwargs): + del args, kwargs + raise AssertionError("dry-run must not load configuration or provider keys") + + async def forbidden_async(*args, **kwargs): + del args, kwargs + raise AssertionError("dry-run must not touch provider or transport seams") + + monkeypatch.setattr(config_module, "load_config", forbidden) + monkeypatch.setattr(registry_module, "key_present", forbidden) + monkeypatch.setattr(live_module, "call_model", forbidden_async) + monkeypatch.setattr(transport_module, "post_json", forbidden_async) + + estimate = await estimate_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + ) + + assert estimate.planned_cells == len(manifest.planned_runs) + + +@pytest.mark.asyncio +async def test_dry_run_reports_calls_costs_largest_reservation_and_headroom( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tasks, manifest, price_book = _live_inputs() + observed_caps: list[int] = [] + observed_costs: list[Decimal] = [] + real_reserve = live_module.reserve_call_cost + + def recording_reserve(*args, **kwargs): + reservation = real_reserve(*args, **kwargs) + observed_caps.append(kwargs["max_output_tokens"]) + observed_costs.append(reservation.reserved_cost_usd) + return reservation + + monkeypatch.setattr(live_module, "reserve_call_cost", recording_reserve) + + estimate = await estimate_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + ) + + roster_sizes = { + roster.roster_id: len(roster.members) for roster in manifest.frozen_design.rosters + } + expected_caps = tuple( + cap + for planned_run in manifest.planned_runs + for cap in allocate_stage_caps( + planned_run.condition_id, + roster_size=roster_sizes[planned_run.roster_id], + cell_ceiling=planned_run.max_output_tokens, + ) + ) + expected_calls = sum( + len( + stage_call_sequence( + planned_run.condition_id, + roster_size=roster_sizes[planned_run.roster_id], + ) + ) + for planned_run in manifest.planned_runs + ) + + assert tuple(observed_caps) == expected_caps + assert estimate.maximum_call_count == expected_calls == len(observed_costs) + assert estimate.largest_reservation_usd == max(observed_costs) + assert estimate.total_upper_bound_usd == sum(observed_costs, Decimal("0")) + assert estimate.ceiling_usd == Decimal("10.00") + assert estimate.headroom_usd == estimate.ceiling_usd - estimate.total_upper_bound_usd + assert estimate.fits_ceiling is True + with pytest.raises(ValidationError): + estimate.maximum_call_count = 0 + + +@pytest.mark.asyncio +async def test_dry_run_breaks_down_upper_bound_by_roster_and_condition() -> None: + tasks, manifest, price_book = _live_inputs() + + estimate = await estimate_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + ) + + assert set(estimate.per_roster_upper_bound_usd) == { + roster.roster_id for roster in manifest.frozen_design.rosters + } + assert set(estimate.per_condition_upper_bound_usd) == { + planned_run.condition_id for planned_run in manifest.planned_runs + } + assert sum(estimate.per_roster_upper_bound_usd.values(), Decimal("0")) == ( + estimate.total_upper_bound_usd + ) + assert sum(estimate.per_condition_upper_bound_usd.values(), Decimal("0")) == ( + estimate.total_upper_bound_usd + ) + assert all(cost > 0 for cost in estimate.per_roster_upper_bound_usd.values()) + assert all(cost > 0 for cost in estimate.per_condition_upper_bound_usd.values()) + + +@pytest.mark.asyncio +async def test_dry_run_rejects_plan_whose_worst_case_exceeds_frozen_ceiling() -> None: + tasks, manifest, price_book = _live_inputs(rate="100000000") + + with pytest.raises(BudgetExceededError, match="worst-case.*frozen ceiling"): + await estimate_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + ) From bbd5c7b454b1ccb98ce294b1cffa153bee12e90f Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 21:15:59 -0400 Subject: [PATCH 08/14] feat(cli): gate live evaluation execution --- src/conclave/eval_cli.py | 144 ++++++++++++++- tests/evals/test_eval_cli.py | 333 +++++++++++++++++++++++++++++++++++ 2 files changed, 472 insertions(+), 5 deletions(-) diff --git a/src/conclave/eval_cli.py b/src/conclave/eval_cli.py index 6fff97c..1600fbd 100644 --- a/src/conclave/eval_cli.py +++ b/src/conclave/eval_cli.py @@ -1,11 +1,13 @@ -"""Experimental, offline-only command surface for the H1 evaluation harness.""" +"""Experimental command surface for the H1 evaluation harness.""" from __future__ import annotations +import asyncio import json import os import tempfile from collections import Counter +from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Annotated, Any @@ -14,17 +16,29 @@ from .evals.blinding import BlindMap, blind_run_records from .evals.dataset import load_public_tasks -from .evals.models import EVAL_CONDITION_IDS, StudyManifest, StudyRun +from .evals.live import ( + LiveGatewayError, + build_checkpoint_bindings, + estimate_live_study, + load_live_checkpoint, +) +from .evals.models import EVAL_CONDITION_IDS, PublicTask, StudyManifest, StudyRun +from .evals.pricing import PriceBook, load_price_book from .evals.protocols import build_study_manifest from .evals.reporting import write_report_bundle -from .evals.runner import validate_run_records +from .evals.runner import ( + LIVE_HARD_CAP_USD, + RunValidationError, + run_live_study, + validate_run_records, +) from .evals.scoring import AdjudicationRecord, GraderJudgment, score_study app = typer.Typer( add_completion=False, help=( - "Experimental H1 evaluation tools (DSE-708). Offline artifacts only; " - "these commands never call providers." + "Experimental H1 evaluation tools (DSE-708). Commands are offline-only except " + "for live --execute, which requires exact frozen spend approval." ), ) @@ -75,6 +89,126 @@ def _validate_run_summary(study_run: StudyRun) -> None: _abort("run artifact summary does not match its immutable records") +def _load_live_artifacts( + manifest_path: Path, + tasks_path: Path, + price_book_path: Path, +) -> tuple[StudyManifest, list[PublicTask], PriceBook]: + manifest = _validated(StudyManifest, manifest_path) + design = manifest.frozen_design + if ( + manifest.evidence_classification != "paid_exploratory_pilot" + or design is None + or design.evidence_classification != "paid_exploratory_pilot" + ): + _abort("live evaluation requires a paid exploratory frozen manifest") + try: + tasks = load_public_tasks(tasks_path) + price_book = load_price_book(price_book_path, frozen_design=design) + except (OSError, json.JSONDecodeError, ValidationError, ValueError) as exc: + _abort(f"invalid live evaluation artifacts: {exc}") + return manifest, tasks, price_book + + +def _require_live_approval(value: str | None, manifest: StudyManifest) -> None: + if value is None: + _abort("--execute requires an exact --approve-spend-usd approval") + try: + approval = Decimal(value) + except InvalidOperation: + _abort("spend approval must be an exact USD decimal") + design = manifest.frozen_design + if design is None: # pragma: no cover - enforced while loading live artifacts + _abort("live evaluation requires a frozen design") + frozen_ceiling = Decimal(str(design.approved_spend_ceiling_usd)) + if approval != LIVE_HARD_CAP_USD or approval != frozen_ceiling: + _abort("spend approval must exactly match USD 10.00 and the frozen design ceiling") + + +@app.command("live") +def live_command( + manifest_path: Annotated[Path, typer.Argument(exists=True, readable=True)], + tasks_path: Annotated[Path, typer.Argument(exists=True, readable=True)], + price_book_path: Annotated[Path, typer.Argument(exists=True, readable=True)], + output: Annotated[Path, typer.Argument()], + checkpoint: Annotated[Path, typer.Argument()], + receipts: Annotated[Path, typer.Argument()], + approve_spend_usd: Annotated[ + str | None, + typer.Option( + "--approve-spend-usd", + help="Exact frozen USD ceiling; required only with --execute.", + ), + ] = None, + execute: Annotated[ + bool, + typer.Option( + "--execute", + help="Allow the guarded live runner to reach providers.", + ), + ] = False, +) -> None: + """Estimate by default; execute only a frozen USD 10 paid exploratory pilot.""" + + manifest, tasks, price_book = _load_live_artifacts( + manifest_path, + tasks_path, + price_book_path, + ) + if not execute: + try: + estimate = asyncio.run( + estimate_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + ) + ) + except (LiveGatewayError, RunValidationError, ValidationError, ValueError) as exc: + _abort(f"live estimate rejected: {exc}") + typer.echo(json.dumps(estimate.model_dump(mode="json"), sort_keys=True)) + return + + _require_live_approval(approve_spend_usd, manifest) + if len({output.resolve(), checkpoint.resolve(), receipts.resolve()}) != 3: + _abort("run, checkpoint, and receipt artifacts require separate paths") + try: + checkpoint.parent.mkdir(parents=True, exist_ok=True) + study_run = asyncio.run( + run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=checkpoint, + ) + ) + bindings = build_checkpoint_bindings( + manifest, + price_book, + hard_cap_usd=LIVE_HARD_CAP_USD, + ) + final_checkpoint = load_live_checkpoint( + checkpoint, + expected_bindings=bindings, + ) + except (OSError, LiveGatewayError, RunValidationError, ValidationError, ValueError) as exc: + _abort(f"live execution rejected: {exc}") + + _atomic_json(output, study_run.model_dump(mode="json")) + _atomic_json( + receipts, + { + "schema_version": final_checkpoint.schema_version, + "study_id": manifest.study_id, + "checkpoint_hash": final_checkpoint.checkpoint_hash, + "bindings": final_checkpoint.bindings.model_dump(mode="json"), + "committed_cost_usd": str(final_checkpoint.committed_cost_usd), + "receipts": [receipt.model_dump(mode="json") for receipt in final_checkpoint.receipts], + }, + ) + typer.echo(f"Executed live study and wrote {output} with separate receipts at {receipts}") + + @app.command("plan") def plan_command( public_tasks: Annotated[Path, typer.Argument(exists=True, readable=True)], diff --git a/tests/evals/test_eval_cli.py b/tests/evals/test_eval_cli.py index 9440e3a..2361a38 100644 --- a/tests/evals/test_eval_cli.py +++ b/tests/evals/test_eval_cli.py @@ -1,12 +1,25 @@ from __future__ import annotations import json +from decimal import Decimal +import pytest from typer.testing import CliRunner +import conclave.eval_cli as eval_cli_module from conclave.cli import app +from conclave.evals.live import ( + build_checkpoint_bindings, + create_live_checkpoint, + load_live_checkpoint, + write_live_checkpoint, +) +from conclave.evals.live_protocols import stage_call_sequence from conclave.evals.models import PublicTask, RunRecord, StudyRun from conclave.evals.protocols import CONDITION_IDS, build_study_manifest +from conclave.evals.runner import run_live_study as execute_live_study +from conclave.models import ModelAnswer, TokenUsage +from tests.evals.test_live_runner import _live_inputs runner = CliRunner() @@ -51,6 +64,326 @@ def _study_artifacts(tmp_path): return tasks_path, manifest_path, run_path, manifest +def _live_artifacts(tmp_path, **input_overrides): + tmp_path.mkdir(parents=True, exist_ok=True) + tasks, manifest, price_book = _live_inputs(**input_overrides) + tasks_path = tmp_path / "live-tasks.json" + manifest_path = tmp_path / "live-manifest.json" + price_book_path = tmp_path / "live-price-book.json" + _write_json( + tasks_path, + { + "schema_version": "conclave_eval_v1", + "tasks": [task.model_dump(mode="json") for task in tasks], + }, + ) + _write_json(manifest_path, manifest.model_dump(mode="json")) + _write_json(price_book_path, price_book.model_dump(mode="json")) + return tasks, manifest, price_book, tasks_path, manifest_path, price_book_path + + +def _live_command( + manifest_path, + tasks_path, + price_book_path, + output_path, + checkpoint_path, + receipts_path, + *options, +) -> list[str]: + return [ + "eval", + "live", + str(manifest_path), + str(tasks_path), + str(price_book_path), + str(output_path), + str(checkpoint_path), + str(receipts_path), + *options, + ] + + +async def _successful_live_provider(name, model_id, messages, **kwargs): + del messages, kwargs + return ModelAnswer( + name=name, + model_id=model_id, + answer=f"fictional decision from {name}", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + +def test_eval_live_defaults_to_dry_run_and_never_calls_provider( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, manifest, _, tasks_path, manifest_path, price_book_path = _live_artifacts(tmp_path) + output_path = tmp_path / "run.json" + checkpoint_path = tmp_path / "checkpoint.json" + receipts_path = tmp_path / "receipts.json" + provider_calls = 0 + + async def forbidden_live_runner(**kwargs): + nonlocal provider_calls + del kwargs + provider_calls += 1 + raise AssertionError("dry-run must never enter the provider-backed runner") + + monkeypatch.setattr(eval_cli_module, "run_live_study", forbidden_live_runner, raising=False) + + result = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + output_path, + checkpoint_path, + receipts_path, + "--approve-spend-usd", + "10.00", + ), + ) + + assert result.exit_code == 0, result.output + estimate = json.loads(result.stdout) + assert estimate["planned_cells"] == len(manifest.planned_runs) + assert Decimal(estimate["ceiling_usd"]) == Decimal("10.00") + assert estimate["fits_ceiling"] is True + assert provider_calls == 0 + assert not output_path.exists() + assert not checkpoint_path.exists() + assert not receipts_path.exists() + + +def test_eval_live_requires_execute_and_exact_frozen_spend_approval( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, _, _, tasks_path, manifest_path, price_book_path = _live_artifacts(tmp_path) + provider_calls = 0 + + async def forbidden_live_runner(**kwargs): + nonlocal provider_calls + del kwargs + provider_calls += 1 + raise AssertionError("invalid approval must fail before provider execution") + + monkeypatch.setattr(eval_cli_module, "run_live_study", forbidden_live_runner, raising=False) + + for index, options in enumerate( + ( + ("--execute",), + ("--execute", "--approve-spend-usd", "9.99"), + ("--execute", "--approve-spend-usd", "10.01"), + ) + ): + result = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + tmp_path / f"run-{index}.json", + tmp_path / f"checkpoint-{index}.json", + tmp_path / f"receipts-{index}.json", + *options, + ), + ) + + assert result.exit_code == 2 + assert "approval" in result.output.lower() + + dry_run = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + tmp_path / "dry-run.json", + tmp_path / "dry-checkpoint.json", + tmp_path / "dry-receipts.json", + "--approve-spend-usd", + "10.00", + ), + ) + assert dry_run.exit_code == 0, dry_run.output + assert provider_calls == 0 + + +def test_eval_live_rejects_confirmatory_legacy_or_snapshot_drift( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, _, _, live_tasks_path, live_manifest_path, live_price_book_path = _live_artifacts( + tmp_path / "valid" + ) + _, _, _, confirmatory_tasks, confirmatory_manifest, confirmatory_price_book = _live_artifacts( + tmp_path / "confirmatory", evidence_classification="confirmatory" + ) + legacy_dir = tmp_path / "legacy" + legacy_dir.mkdir() + legacy_tasks, legacy_manifest, _, _ = _study_artifacts(legacy_dir) + drifted_payload = json.loads(live_price_book_path.read_text(encoding="utf-8")) + drifted_payload["snapshot_id"] = "drifted-snapshot" + drifted_price_book = tmp_path / "drifted-price-book.json" + _write_json(drifted_price_book, drifted_payload) + boundary_calls = 0 + + async def forbidden_boundary(**kwargs): + nonlocal boundary_calls + del kwargs + boundary_calls += 1 + raise AssertionError("invalid live artifacts must fail before keys or network") + + monkeypatch.setattr(eval_cli_module, "estimate_live_study", forbidden_boundary, raising=False) + monkeypatch.setattr(eval_cli_module, "run_live_study", forbidden_boundary, raising=False) + + cases = ( + ( + confirmatory_manifest, + confirmatory_tasks, + confirmatory_price_book, + "paid exploratory", + ), + (legacy_manifest, legacy_tasks, live_price_book_path, "paid exploratory"), + (live_manifest_path, live_tasks_path, drifted_price_book, "snapshot"), + ) + for index, (manifest_path, tasks_path, price_book_path, expected) in enumerate(cases): + result = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + tmp_path / f"invalid-run-{index}.json", + tmp_path / f"invalid-checkpoint-{index}.json", + tmp_path / f"invalid-receipts-{index}.json", + "--execute", + "--approve-spend-usd", + "10.00", + ), + ) + + assert result.exit_code == 2 + assert expected in result.output.lower() + + assert boundary_calls == 0 + + +def test_eval_live_writes_checkpoint_receipts_and_study_run_atomically( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, manifest, _, tasks_path, manifest_path, price_book_path = _live_artifacts(tmp_path) + output_path = tmp_path / "artifacts" / "run.json" + checkpoint_path = tmp_path / "state" / "checkpoint.json" + receipts_path = tmp_path / "artifacts" / "receipts.json" + atomic_paths = [] + real_atomic_json = eval_cli_module._atomic_json + + async def injected_runner(**kwargs): + return await execute_live_study( + **kwargs, + call_model_func=_successful_live_provider, + ) + + def recording_atomic_json(path, value): + atomic_paths.append(path) + real_atomic_json(path, value) + + monkeypatch.setattr(eval_cli_module, "run_live_study", injected_runner, raising=False) + monkeypatch.setattr(eval_cli_module, "_atomic_json", recording_atomic_json) + + result = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + output_path, + checkpoint_path, + receipts_path, + "--execute", + "--approve-spend-usd", + "10.00", + ), + ) + + assert result.exit_code == 0, result.output + run_payload = json.loads(output_path.read_text(encoding="utf-8")) + receipt_payload = json.loads(receipts_path.read_text(encoding="utf-8")) + assert run_payload["total_planned_runs"] == len(manifest.planned_runs) + assert receipt_payload["study_id"] == manifest.study_id + assert receipt_payload["receipts"] + assert checkpoint_path.exists() + assert atomic_paths == [output_path, receipts_path] + assert not tuple(tmp_path.rglob("*.tmp")) + + +def test_eval_live_resume_uses_existing_checkpoint( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, manifest, price_book, tasks_path, manifest_path, price_book_path = _live_artifacts(tmp_path) + output_path = tmp_path / "run.json" + checkpoint_path = tmp_path / "checkpoint.json" + receipts_path = tmp_path / "receipts.json" + first_run = manifest.planned_runs[0] + preserved = RunRecord( + planned_run_id=first_run.planned_run_id, + outcome="success", + output="preserved checkpoint result", + cost_receipt_complete=True, + ) + bindings = build_checkpoint_bindings( + manifest, + price_book, + hard_cap_usd=Decimal("10.00"), + ) + write_live_checkpoint( + checkpoint_path, + create_live_checkpoint(bindings=bindings, records=(preserved,)), + ) + provider_calls = 0 + + async def counting_provider(name, model_id, messages, **kwargs): + nonlocal provider_calls + provider_calls += 1 + return await _successful_live_provider(name, model_id, messages, **kwargs) + + async def injected_runner(**kwargs): + assert kwargs["checkpoint_path"] == checkpoint_path + return await execute_live_study(**kwargs, call_model_func=counting_provider) + + monkeypatch.setattr(eval_cli_module, "run_live_study", injected_runner, raising=False) + + result = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + output_path, + checkpoint_path, + receipts_path, + "--execute", + "--approve-spend-usd", + "10.00", + ), + ) + + assert result.exit_code == 0, result.output + run_payload = json.loads(output_path.read_text(encoding="utf-8")) + assert run_payload["records"][0]["output"] == "preserved checkpoint result" + expected_calls = sum( + len(stage_call_sequence(item.condition_id, roster_size=3)) + for item in manifest.planned_runs[1:] + ) + assert provider_calls == expected_calls + resumed_checkpoint = load_live_checkpoint( + checkpoint_path, + expected_bindings=bindings, + ) + assert resumed_checkpoint.records[0] == preserved + + def test_eval_plan_writes_complete_manifest_atomically(tmp_path) -> None: tasks_path, _, _, _ = _study_artifacts(tmp_path) output = tmp_path / "nested" / "manifest.json" From a6346c4588f4bd4492c395a1cf377d51a5298d58 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 21:26:34 -0400 Subject: [PATCH 09/14] test(evals): replay capped live conditions --- tests/evals/test_live_replay.py | 250 ++ tests/evals/test_replay.py | 6 + tests/fixtures/evals/live_smoke/manifest.json | 272 ++ .../evals/live_smoke/public_tasks.json | 17 + tests/fixtures/evals/live_smoke/replay.json | 2230 +++++++++++++++++ 5 files changed, 2775 insertions(+) create mode 100644 tests/evals/test_live_replay.py create mode 100644 tests/fixtures/evals/live_smoke/manifest.json create mode 100644 tests/fixtures/evals/live_smoke/public_tasks.json create mode 100644 tests/fixtures/evals/live_smoke/replay.json diff --git a/tests/evals/test_live_replay.py b/tests/evals/test_live_replay.py new file mode 100644 index 0000000..2ac494f --- /dev/null +++ b/tests/evals/test_live_replay.py @@ -0,0 +1,250 @@ +"""End-to-end sanitized transport replay for the capped live condition matrix.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import conclave.evals.runner as runner_module +import conclave.transport as transport +from conclave.config import ConclaveConfig, CustomEndpoint +from conclave.evals.dataset import load_public_tasks +from conclave.evals.live import ( + build_checkpoint_bindings, + hash_study_manifest, + load_live_checkpoint, +) +from conclave.evals.live_protocols import stage_call_sequence +from conclave.evals.models import EVAL_CONDITION_IDS, StudyManifest +from conclave.evals.pricing import PriceBook +from conclave.evals.replay import ( + RecordingPostJson, + ReplayArtifact, + ReplayingPostJson, + ReplayMismatchError, +) +from conclave.evals.runner import LIVE_HARD_CAP_USD, run_live_study +from conclave.providers import call_model + +FIXTURE_DIR = Path(__file__).parents[1] / "fixtures/evals/live_smoke" +FAKE_KEY_ENV = "CONCLAVE_FAKE_TEST_KEY" +FAKE_KEY = "fixture-only-opaque-credential" + + +def _canonical_bytes(value: object) -> bytes: + if hasattr(value, "model_dump"): + value = value.model_dump(mode="json") + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode() + + +def _deterministic_clock(): + tick = -1 + + def perf_counter() -> float: + nonlocal tick + tick += 1 + return tick / 1000 + + return perf_counter + + +def _custom_config(manifest: StudyManifest) -> ConclaveConfig: + model_ids = { + member.model_id for roster in manifest.frozen_design.rosters for member in roster.members + } + return ConclaveConfig( + endpoints={ + model_id: CustomEndpoint( + completions_url="https://fictional.invalid/v1/chat/completions", + env_var=FAKE_KEY_ENV, + ) + for model_id in model_ids + } + ) + + +def _fake_delegate(): + calls: list[tuple[str, str, int]] = [] + + async def delegate(url, headers, body, timeout): + assert url == "https://fictional.invalid/v1/chat/completions" + assert headers["Authorization"] == f"Bearer {FAKE_KEY}" + calls.append((url, body["model"], body["max_tokens"])) + ordinal = len(calls) + return 200, { + "choices": [{"message": {"content": f"Fictional decision artifact {ordinal:03d}."}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + return delegate, calls + + +async def _execute( + *, + monkeypatch, + manifest, + tasks, + price_book, + post_json, + checkpoint_path, +): + config = _custom_config(manifest) + + async def configured_call_model(name, model_id, messages, **kwargs): + return await call_model(name, model_id, messages, config=config, **kwargs) + + monkeypatch.setattr(transport, "post_json", post_json) + monkeypatch.setattr( + runner_module, + "time", + SimpleNamespace(perf_counter=_deterministic_clock()), + ) + study_run = await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=checkpoint_path, + call_model_func=configured_call_model, + ) + checkpoint = load_live_checkpoint( + checkpoint_path, + expected_bindings=build_checkpoint_bindings( + manifest, + price_book, + hard_cap_usd=LIVE_HARD_CAP_USD, + ), + ) + return study_run, checkpoint + + +@pytest.mark.asyncio +async def test_live_smoke_replay_executes_all_conditions_with_zero_network_calls( + tmp_path, monkeypatch, clear_keys +) -> None: + del clear_keys + monkeypatch.setenv(FAKE_KEY_ENV, FAKE_KEY) + tasks = load_public_tasks(FIXTURE_DIR / "public_tasks.json") + manifest = StudyManifest.model_validate_json( + (FIXTURE_DIR / "manifest.json").read_text(encoding="utf-8") + ) + price_book = PriceBook.model_validate_json( + (FIXTURE_DIR / "price_book.json").read_text(encoding="utf-8") + ) + artifact = ReplayArtifact.model_validate_json( + (FIXTURE_DIR / "replay.json").read_text(encoding="utf-8") + ) + base_manifest_hash = hash_study_manifest(manifest) + assert artifact.base_manifest_hash == base_manifest_hash + + delegate, delegate_calls = _fake_delegate() + recorder = RecordingPostJson(delegate, base_manifest_hash=base_manifest_hash) + recorded_run, recorded_checkpoint = await _execute( + monkeypatch=monkeypatch, + manifest=manifest, + tasks=tasks, + price_book=price_book, + post_json=recorder, + checkpoint_path=tmp_path / "recorded-checkpoint.json", + ) + assert _canonical_bytes(recorder.artifact()) == _canonical_bytes(artifact) + + replay_one = ReplayingPostJson(artifact, base_manifest_hash=base_manifest_hash) + first_run, first_checkpoint = await _execute( + monkeypatch=monkeypatch, + manifest=manifest, + tasks=tasks, + price_book=price_book, + post_json=replay_one, + checkpoint_path=tmp_path / "first-replay-checkpoint.json", + ) + replay_one.assert_consumed() + + replay_two = ReplayingPostJson(artifact, base_manifest_hash=base_manifest_hash) + second_run, second_checkpoint = await _execute( + monkeypatch=monkeypatch, + manifest=manifest, + tasks=tasks, + price_book=price_book, + post_json=replay_two, + checkpoint_path=tmp_path / "second-replay-checkpoint.json", + ) + replay_two.assert_consumed() + + planned_ids = [planned.planned_run_id for planned in manifest.planned_runs] + record_counts = Counter(record.planned_run_id for record in first_run.records) + assert record_counts == Counter(dict.fromkeys(planned_ids, 1)) + assert all(record.outcome == "success" for record in first_run.records) + condition_counts = Counter(planned.condition_id for planned in manifest.planned_runs) + assert set(condition_counts) == set(EVAL_CONDITION_IDS) + assert set(condition_counts.values()) == {len(manifest.frozen_design.rosters)} + + roster_by_id = {roster.roster_id: roster for roster in manifest.frozen_design.rosters} + expected_call_order = [] + for planned in manifest.planned_runs: + roster = roster_by_id[planned.roster_id] + expected_call_order.extend( + (stage, roster.members[index].provider_id, roster.members[index].model_id) + for stage, index in stage_call_sequence( + planned.condition_id, + roster_size=len(roster.members), + ) + ) + actual_call_order = [ + (receipt.stage, receipt.provider_id, receipt.model_id) + for receipt in first_checkpoint.receipts + ] + assert actual_call_order == expected_call_order + assert len(delegate_calls) == len(expected_call_order) == len(artifact.records) + + def receipt_bytes(checkpoint): + return _canonical_bytes( + [receipt.model_dump(mode="json") for receipt in checkpoint.receipts] + ) + + assert _canonical_bytes(recorded_run) == _canonical_bytes(first_run) + assert _canonical_bytes(first_run) == _canonical_bytes(second_run) + assert receipt_bytes(recorded_checkpoint) == receipt_bytes(first_checkpoint) + assert receipt_bytes(first_checkpoint) == receipt_bytes(second_checkpoint) + assert (tmp_path / "first-replay-checkpoint.json").read_bytes() == ( + tmp_path / "second-replay-checkpoint.json" + ).read_bytes() + + serialized = b"\n".join( + ( + (FIXTURE_DIR / "replay.json").read_bytes(), + (tmp_path / "recorded-checkpoint.json").read_bytes(), + (tmp_path / "first-replay-checkpoint.json").read_bytes(), + receipt_bytes(first_checkpoint), + _canonical_bytes(first_run), + ) + ) + assert FAKE_KEY.encode() not in serialized + + delegate_count = len(delegate_calls) + missing = ReplayingPostJson(artifact, base_manifest_hash=base_manifest_hash) + with pytest.raises(ReplayMismatchError, match="unconsumed record"): + missing.assert_consumed() + + first_record = artifact.records[0] + changed_body = {**first_record.request["body"], "temperature": 0.125} + changed = ReplayingPostJson(artifact, base_manifest_hash=base_manifest_hash) + with pytest.raises(ReplayMismatchError, match="unmatched request"): + await changed( + first_record.request["url"], + {"Authorization": f"Bearer {FAKE_KEY}"}, + changed_body, + 1, + ) + + with pytest.raises(ReplayMismatchError, match="unmatched request"): + await replay_one( + first_record.request["url"], + {"Authorization": f"Bearer {FAKE_KEY}"}, + first_record.request["body"], + 1, + ) + assert len(delegate_calls) == delegate_count diff --git a/tests/evals/test_replay.py b/tests/evals/test_replay.py index 5409c7a..3af290c 100644 --- a/tests/evals/test_replay.py +++ b/tests/evals/test_replay.py @@ -130,7 +130,11 @@ def test_replay_rejects_schema_or_base_manifest_hash_drift(): async def test_replay_fails_closed_on_missing_mismatch_and_extra_records(): + network_calls = 0 + async def network(url, headers, body, timeout): + nonlocal network_calls + network_calls += 1 return 200, {"ok": True} recorder = RecordingPostJson(network, base_manifest_hash=BASE_HASH) @@ -150,6 +154,8 @@ async def network(url, headers, body, timeout): with pytest.raises(ReplayMismatchError, match="unmatched request"): await replay("https://example.test/v1", {}, {"model": "m", "prompt": "one"}, 10) + assert network_calls == 1 + def test_replay_artifact_rejects_stored_request_hash_mismatch(): request = {"url": "https://example.test/v1", "body": {"prompt": "safe"}} diff --git a/tests/fixtures/evals/live_smoke/manifest.json b/tests/fixtures/evals/live_smoke/manifest.json new file mode 100644 index 0000000..9161f00 --- /dev/null +++ b/tests/fixtures/evals/live_smoke/manifest.json @@ -0,0 +1,272 @@ +{ + "schema_version": "conclave_eval_v1", + "study_id": "fictional-live-smoke", + "evidence_classification": "paid_exploratory_pilot", + "promotable": false, + "seed": 20260718, + "replicates": 1, + "task_ids": [ + "fictional-live-task" + ], + "public_tasks_hash": "sha256:d57261e507a99d710e65f0717c6ba18258c2de25d6ab0c0badbab143f7688e05", + "frozen_design": { + "schema_version": "conclave_eval_v1", + "evidence_classification": "paid_exploratory_pilot", + "base_commit": "2222222222222222222222222222222222222222", + "task_family_map": { + "fictional-live-task": "fictional-safety-choice" + }, + "rosters": [ + { + "schema_version": "conclave_eval_v1", + "roster_id": "fictional-roster-forward", + "members": [ + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-a", + "model_id": "fictional-model-a", + "model_revision": "fixture-r1" + }, + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-b", + "model_id": "fictional-model-b", + "model_revision": "fixture-r2" + }, + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-c", + "model_id": "fictional-model-c", + "model_revision": "fixture-r3" + } + ] + }, + { + "schema_version": "conclave_eval_v1", + "roster_id": "fictional-roster-reverse", + "members": [ + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-c", + "model_id": "fictional-model-c", + "model_revision": "fixture-r3" + }, + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-b", + "model_id": "fictional-model-b", + "model_revision": "fixture-r2" + }, + { + "schema_version": "conclave_eval_v1", + "provider_id": "fictional-provider-a", + "model_id": "fictional-model-a", + "model_revision": "fixture-r1" + } + ] + } + ], + "condition_prompt_versions": { + "single_frontier": "live_single_frontier_prompt_v1", + "self_refine": "live_self_refine_prompt_v1", + "independent_synthesis": "live_synthesis_2026-06-29", + "critique_only": "live_critique_1_synthesis_2026-06-29", + "revision_only": "live_revision_v1_synthesis_2026-06-29", + "elite_full": "elite_1_synthesis_2026-06-29_verdict_2" + }, + "condition_protocol_versions": { + "single_frontier": "single_frontier_live_v1", + "self_refine": "self_refine_live_v1", + "independent_synthesis": "independent_synthesis_live_v1", + "critique_only": "critique_only_live_v1", + "revision_only": "revision_only_live_v1", + "elite_full": "elite_full_live_v1" + }, + "generation_settings_hash": "sha256:3333333333333333333333333333333333333333333333333333333333333333", + "evaluator_version": "fictional-live-evaluator-v1", + "analysis_code_hash": "sha256:4444444444444444444444444444444444444444444444444444444444444444", + "rubric_hash": "sha256:5555555555555555555555555555555555555555555555555555555555555555", + "grader_instructions_hash": "sha256:6666666666666666666666666666666666666666666666666666666666666666", + "grader_keys_hash": "sha256:7777777777777777777777777777777777777777777777777777777777777777", + "exclusion_deviation_policy": { + "schema_version": "conclave_eval_v1", + "predefined_task_exclusions": [], + "excluded_task_ids": [], + "output_level_exclusions_allowed": false, + "deviations_remain_in_denominator": true + }, + "timeout_retry_policy": { + "schema_version": "conclave_eval_v1", + "timeout_seconds": 1.0, + "retry_attempts": 0, + "exhausted_runs_remain_in_denominator": true + }, + "randomization": { + "schema_version": "conclave_eval_v1", + "master_seed": 20260718, + "method": "sha256_task_roster_block_v1" + }, + "bootstrap": { + "schema_version": "conclave_eval_v1", + "seed": 20260718, + "samples": 10, + "confidence_level": 0.95, + "unit": "task" + }, + "analysis_gates": { + "schema_version": "conclave_eval_v1", + "primary_baseline": "single_frontier", + "alpha": 0.05, + "power": 0.8, + "minimum_effect": 0.1, + "severe_error_noninferiority_margin": 0.02, + "readiness_noninferiority_margin": 0.05, + "reviewer_effort_max_ratio": 1.2, + "reviewer_effort_statistic": "ratio_of_task_medians", + "latency_baseline": "single_frontier", + "p95_latency_max_ratio": 3.0, + "absolute_p95_latency_seconds": 60.0, + "minimum_confirmatory_tasks": 2, + "minimum_double_grading_rate": 0.95, + "minimum_raw_agreement": 0.8, + "minimum_overall_kappa": 0.6, + "minimum_family_kappa": 0.5, + "maximum_adjudication_rate": 0.2, + "multiplicity_rule": "bonferroni_simultaneous_upper_bounds", + "unit": "task" + }, + "price_snapshot": { + "schema_version": "conclave_eval_v1", + "snapshot_id": "fictional-live-smoke-prices-2026-07-18", + "captured_at": "2026-07-18T12:00:00Z", + "currency": "USD", + "prices_hash": "sha256:df85e17c12985ada8a350c6f2adc1778a0218bf3dc70cf3204ac8d1057939aac" + }, + "approved_spend_ceiling_usd": 10.0, + "preregistration_id": null, + "preregistration_hash": null + }, + "frozen_design_hash": "sha256:7b8fc1a68dd70af8538d61af258d84dfb16f97b744b8fa584b45a12ea352e9d0", + "planned_runs": [ + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_c89136045f317beb2776b8a7", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-forward", + "condition_id": "revision_only", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_26d60cf8204277ed433cf693", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-forward", + "condition_id": "elite_full", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_2d94e5980d36edfdd6f924ef", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-forward", + "condition_id": "independent_synthesis", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_6354ef6a4d3a3d6af3b0ba5b", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-forward", + "condition_id": "self_refine", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_0677ca748602fc84dd97a6f2", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-forward", + "condition_id": "critique_only", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_fbf0a2094bd7071b3fd15766", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-forward", + "condition_id": "single_frontier", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_12f47bc053d373a70c36e190", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-reverse", + "condition_id": "elite_full", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_22980c01680221e2a1af883f", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-reverse", + "condition_id": "self_refine", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_2d41f152248bbbe7b0a872f3", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-reverse", + "condition_id": "revision_only", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_5aa1617957acbe77c6e416a1", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-reverse", + "condition_id": "critique_only", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_c0acdec7f0d30a4b7db4e7b5", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-reverse", + "condition_id": "independent_synthesis", + "replicate": 1, + "max_output_tokens": 64 + }, + { + "schema_version": "conclave_eval_v1", + "planned_run_id": "run_9354497a641db13eb5d91cdd", + "study_id": "fictional-live-smoke", + "task_id": "fictional-live-task", + "roster_id": "fictional-roster-reverse", + "condition_id": "single_frontier", + "replicate": 1, + "max_output_tokens": 64 + } + ] +} diff --git a/tests/fixtures/evals/live_smoke/public_tasks.json b/tests/fixtures/evals/live_smoke/public_tasks.json new file mode 100644 index 0000000..055364e --- /dev/null +++ b/tests/fixtures/evals/live_smoke/public_tasks.json @@ -0,0 +1,17 @@ +{ + "schema_version": "conclave_eval_v1", + "tasks": [ + { + "schema_version": "conclave_eval_v1", + "task_id": "fictional-live-task", + "prompt": "Select the safest route for a fictional lunar archive migration.", + "reference_packets": [ + "Fictional archive route A has a tested rollback procedure.", + "All systems, organizations, risks, and costs in this task are fictional." + ], + "metadata": { + "fixture": "fictional-only" + } + } + ] +} diff --git a/tests/fixtures/evals/live_smoke/replay.json b/tests/fixtures/evals/live_smoke/replay.json new file mode 100644 index 0000000..c38778d --- /dev/null +++ b/tests/fixtures/evals/live_smoke/replay.json @@ -0,0 +1,2230 @@ +{ + "schema_version": "conclave_replay_v1", + "base_manifest_hash": "sha256:456096f88d22398553a1d0cc6024c8ed469f9115a28731f30ce4d1d960155442", + "records": [ + { + "request_hash": "sha256:8e116cb7c77c18cbe04219e375cd95923cb30625aec753fbc4a4a3ca2907dea3", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 001." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:f97d7447f59a3a1923b83ecdbd2a256e788aa0306b80e34487c82f33195cc1c8", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 002." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:9a6ce2607e0eb30c7477f008399efde19c5aeed1b7fa1c061620db31b2946fa9", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 003." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:971312b5299013bf65b12468532d0cf1a21dcbfd6f5813b465b5d1e4ed3780f8", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 001.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 001.\n\n### Model B peer answer\nFictional decision artifact 002.\n\n### Model C peer answer\nFictional decision artifact 003.\n\nProduce one complete revised answer to the original prompt.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 004." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:9e438ed4b52535986e2e1e9353d84315ed6390a1af4cba5d49f56ac13c8c8bf8", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 002.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 001.\n\n### Model B peer answer\nFictional decision artifact 002.\n\n### Model C peer answer\nFictional decision artifact 003.\n\nProduce one complete revised answer to the original prompt.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 005." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:7da3f22b52b7040fe3329c7b54d8c9ee33c64d5f129d9f2457d32422d69bb284", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 003.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 001.\n\n### Model B peer answer\nFictional decision artifact 002.\n\n### Model C peer answer\nFictional decision artifact 003.\n\nProduce one complete revised answer to the original prompt.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 006." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:006fcd47eb736ca44e176c41ec3f123db8fc74f74474ff0d859c7ff21e4bf493", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 10, + "messages": [ + { + "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", + "role": "system" + }, + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized decision artifacts:\n\n### Model A decision artifact\nFictional decision artifact 004.\n\n### Model B decision artifact\nFictional decision artifact 005.\n\n### Model C decision artifact\nFictional decision artifact 006.\n\nProduce the consolidated decision artifact.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 007." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:c06913d309bff2c3b15ee8978b1776b0f9bfa776d681d8491e394f0fffa0d41c", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 008." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:84517758489cf8ff0b17c73e5067d71fcc01244d338af04f4ad5bdca2c016df3", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 009." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:3fdbe8d950bdeec566d42580d4f3f09f99a160cdd3ce6b1c4c07386489d2d8cc", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 010." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:50912c3ad490b25f600bfe77424145b215768f2ef862cadf0caffca899e812d2", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_f29fa12f48917da3ddf9db46)\nFictional decision artifact 008.\n\n### Model B initial answer (Answer ID: ca_initial_c9faf8db2a5723c726845328)\nFictional decision artifact 009.\n\n### Model C initial answer (Answer ID: ca_initial_4932305110971a8ba03c3fd7)\nFictional decision artifact 010.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 011." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:89a1c6f02c94f203c449bae2c49321df5738cf3e999cc3f1ab2394990655fabf", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_f29fa12f48917da3ddf9db46)\nFictional decision artifact 008.\n\n### Model B initial answer (Answer ID: ca_initial_c9faf8db2a5723c726845328)\nFictional decision artifact 009.\n\n### Model C initial answer (Answer ID: ca_initial_4932305110971a8ba03c3fd7)\nFictional decision artifact 010.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 012." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:f31454bbd91ab0005217e1c9a12369ea7009cf6d79c9901c904897816c2616c7", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_f29fa12f48917da3ddf9db46)\nFictional decision artifact 008.\n\n### Model B initial answer (Answer ID: ca_initial_c9faf8db2a5723c726845328)\nFictional decision artifact 009.\n\n### Model C initial answer (Answer ID: ca_initial_4932305110971a8ba03c3fd7)\nFictional decision artifact 010.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 013." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:1d7cc1836d4d58b9be1f8fc3db663f862f2aa72012e2a3cebed3d5a801c46224", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer (Answer ID: ca_initial_f29fa12f48917da3ddf9db46):\nFictional decision artifact 008.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_f29fa12f48917da3ddf9db46)\nFictional decision artifact 008.\n\n### Model B initial answer (Answer ID: ca_initial_c9faf8db2a5723c726845328)\nFictional decision artifact 009.\n\n### Model C initial answer (Answer ID: ca_initial_4932305110971a8ba03c3fd7)\nFictional decision artifact 010.\n\nAnonymized claim audits:\n\n### Model A critique (Answer ID: ca_critique_9b84e50fc06aa0e0b0264032)\nFictional decision artifact 011.\n\n### Model B critique (Answer ID: ca_critique_f6dae839699a7d8094440acf)\nFictional decision artifact 012.\n\n### Model C critique (Answer ID: ca_critique_c6a9ad6602a885891dcfaa49)\nFictional decision artifact 013.\n\nNow provide your claim-audit-aware revised answer.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 014." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:316eb69ec8c0cc581685f457d5b52403220660c5246bff6bd1c0e190023544c6", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer (Answer ID: ca_initial_c9faf8db2a5723c726845328):\nFictional decision artifact 009.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_f29fa12f48917da3ddf9db46)\nFictional decision artifact 008.\n\n### Model B initial answer (Answer ID: ca_initial_c9faf8db2a5723c726845328)\nFictional decision artifact 009.\n\n### Model C initial answer (Answer ID: ca_initial_4932305110971a8ba03c3fd7)\nFictional decision artifact 010.\n\nAnonymized claim audits:\n\n### Model A critique (Answer ID: ca_critique_9b84e50fc06aa0e0b0264032)\nFictional decision artifact 011.\n\n### Model B critique (Answer ID: ca_critique_f6dae839699a7d8094440acf)\nFictional decision artifact 012.\n\n### Model C critique (Answer ID: ca_critique_c6a9ad6602a885891dcfaa49)\nFictional decision artifact 013.\n\nNow provide your claim-audit-aware revised answer.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 015." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:f1e1a14b5c6911a5da18d3ce1506e26e9d20383780714fc5346da584c9d96884", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer (Answer ID: ca_initial_4932305110971a8ba03c3fd7):\nFictional decision artifact 010.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_f29fa12f48917da3ddf9db46)\nFictional decision artifact 008.\n\n### Model B initial answer (Answer ID: ca_initial_c9faf8db2a5723c726845328)\nFictional decision artifact 009.\n\n### Model C initial answer (Answer ID: ca_initial_4932305110971a8ba03c3fd7)\nFictional decision artifact 010.\n\nAnonymized claim audits:\n\n### Model A critique (Answer ID: ca_critique_9b84e50fc06aa0e0b0264032)\nFictional decision artifact 011.\n\n### Model B critique (Answer ID: ca_critique_f6dae839699a7d8094440acf)\nFictional decision artifact 012.\n\n### Model C critique (Answer ID: ca_critique_c6a9ad6602a885891dcfaa49)\nFictional decision artifact 013.\n\nNow provide your claim-audit-aware revised answer.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 016." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:fe72eae35cdb78ab15a438a081f37c978b8852489cd369510aebc26af5f4ed35", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", + "role": "system" + }, + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized decision artifacts:\n\n### Model A decision artifact\nFictional decision artifact 014.\n\n### Model B decision artifact\nFictional decision artifact 015.\n\n### Model C decision artifact\nFictional decision artifact 016.\n\nProduce the consolidated decision artifact.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 017." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:85c6a8f75e6d3685ec036be0e01fc80aa0835cf817f57286110971c4987370cd", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 14, + "messages": [ + { + "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Add conflicts[] only when there are two or more positions in tension.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number \u2014 the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_30f27d6d1f3068239a264a92) \u2014 from Model A\nFictional decision artifact 014.\n\n### Member answer (answer id: ca_revision_54bce3f164f68c9ddb3effdf) \u2014 from Model B\nFictional decision artifact 015.\n\n### Member answer (answer id: ca_revision_648c3c5d70c342eaa5812f6e) \u2014 from Model C\nFictional decision artifact 016.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"consensus_score\": {\n \"type\": [\n \"number\",\n \"null\"\n ]\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n },\n \"confidence\": {\n \"type\": \"string\",\n \"enum\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"verdict_applies\"\n ]\n}", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 018." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:66b06f34e69ffb9fa4bfaa79969147bd90b1c424dfd19a950c80c57a953eeb0b", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 16, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 019." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:cfabe1fcf9f9aa97f5ce3c6f5fd00c0b58e0d19b6328b0858efe8559d145ca8a", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 16, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 020." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:64c25574dc75e5cb8785b3accba9028868bff1e40e1f871b2c7dff341d1958a1", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 16, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 021." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:c4806c924f37455be442b090adf8b2137c31e00568c74b22d006a27465541af4", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 16, + "messages": [ + { + "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", + "role": "system" + }, + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized decision artifacts:\n\n### Model A decision artifact\nFictional decision artifact 019.\n\n### Model B decision artifact\nFictional decision artifact 020.\n\n### Model C decision artifact\nFictional decision artifact 021.\n\nProduce the consolidated decision artifact.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 022." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:ad331e55c84a2f6c683a6bae288410db79d8e2a96bb5eaafec8e77a16b6be353", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 32, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 023." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:af1a0c246e565e9909111e9dc104f3e2c90de91a9009607c3b32e848383e60c6", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 32, + "messages": [ + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour draft:\nFictional decision artifact 023.\n\nProduce one complete, improved final answer.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 024." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:8e116cb7c77c18cbe04219e375cd95923cb30625aec753fbc4a4a3ca2907dea3", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 025." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:f97d7447f59a3a1923b83ecdbd2a256e788aa0306b80e34487c82f33195cc1c8", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 026." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:9a6ce2607e0eb30c7477f008399efde19c5aeed1b7fa1c061620db31b2946fa9", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 027." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:a1f4e1656f2834e946475854cb1f9ac9088f44faca5f545fd3cf2da08d56432a", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_5127d3c1427487a9d5a2105e)\nFictional decision artifact 025.\n\n### Model B initial answer (Answer ID: ca_initial_8d80d7037e094de16cf8580e)\nFictional decision artifact 026.\n\n### Model C initial answer (Answer ID: ca_initial_3c118037d4b7a6632c6b0a09)\nFictional decision artifact 027.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 028." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:4a884b70e16a4838b8a7113c53fa069413cf91549de543d3f6acadb2eacf8510", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_5127d3c1427487a9d5a2105e)\nFictional decision artifact 025.\n\n### Model B initial answer (Answer ID: ca_initial_8d80d7037e094de16cf8580e)\nFictional decision artifact 026.\n\n### Model C initial answer (Answer ID: ca_initial_3c118037d4b7a6632c6b0a09)\nFictional decision artifact 027.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 029." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:46dcb70a5b05062e46f157a91a70efe730961577dbddbbb5382440e5eaa3f17e", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_5127d3c1427487a9d5a2105e)\nFictional decision artifact 025.\n\n### Model B initial answer (Answer ID: ca_initial_8d80d7037e094de16cf8580e)\nFictional decision artifact 026.\n\n### Model C initial answer (Answer ID: ca_initial_3c118037d4b7a6632c6b0a09)\nFictional decision artifact 027.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 030." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:a9b39c9034341d9a625f1a1369f341650c55f188a3da89ccaf3ebb62b17eb6f8", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 10, + "messages": [ + { + "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", + "role": "system" + }, + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized decision artifacts:\n\n### Model A decision artifact\nFictional decision artifact 025.\n\n### Model B decision artifact\nFictional decision artifact 026.\n\n### Model C decision artifact\nFictional decision artifact 027.\n\nAnonymized claim audits:\n\n### Model A anonymized audit\nFictional decision artifact 028.\n\n### Model B anonymized audit\nFictional decision artifact 029.\n\n### Model C anonymized audit\nFictional decision artifact 030.\n\nProduce the consolidated decision artifact.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 031." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:6fad3d158dc6fbf6b5d4ec4ee9206bc152d6e18d4aecde8b6c7b5997db5bf47f", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 64, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 032." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:3fdbe8d950bdeec566d42580d4f3f09f99a160cdd3ce6b1c4c07386489d2d8cc", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 033." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:84517758489cf8ff0b17c73e5067d71fcc01244d338af04f4ad5bdca2c016df3", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 034." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:c06913d309bff2c3b15ee8978b1776b0f9bfa776d681d8491e394f0fffa0d41c", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 035." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:52b77096e212f4eecdbdcb75ff2bca28fb0f8b2c686ffff6fd11dea29da0807d", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_1625c335ddfd9096184d4f8c)\nFictional decision artifact 033.\n\n### Model B initial answer (Answer ID: ca_initial_7aa7c02530241beed834c517)\nFictional decision artifact 034.\n\n### Model C initial answer (Answer ID: ca_initial_e079f32b25f027c8193b28f2)\nFictional decision artifact 035.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 036." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:a835c8acb9a3f1406ba7d7d7ab59cb1db5c4b1c2aae78ecb1388742c5bcef071", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_1625c335ddfd9096184d4f8c)\nFictional decision artifact 033.\n\n### Model B initial answer (Answer ID: ca_initial_7aa7c02530241beed834c517)\nFictional decision artifact 034.\n\n### Model C initial answer (Answer ID: ca_initial_e079f32b25f027c8193b28f2)\nFictional decision artifact 035.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 037." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:8fe9864ae4f5edff442dcc1056248af7878d4aa676744d94f08d252b49c2b080", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_1625c335ddfd9096184d4f8c)\nFictional decision artifact 033.\n\n### Model B initial answer (Answer ID: ca_initial_7aa7c02530241beed834c517)\nFictional decision artifact 034.\n\n### Model C initial answer (Answer ID: ca_initial_e079f32b25f027c8193b28f2)\nFictional decision artifact 035.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 038." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:9259e556a6b7ccff0fb00fa19df61c2ceac242d99ac3823083c166e32ca4e2fb", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer (Answer ID: ca_initial_1625c335ddfd9096184d4f8c):\nFictional decision artifact 033.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_1625c335ddfd9096184d4f8c)\nFictional decision artifact 033.\n\n### Model B initial answer (Answer ID: ca_initial_7aa7c02530241beed834c517)\nFictional decision artifact 034.\n\n### Model C initial answer (Answer ID: ca_initial_e079f32b25f027c8193b28f2)\nFictional decision artifact 035.\n\nAnonymized claim audits:\n\n### Model A critique (Answer ID: ca_critique_3da5202b85c8c8b44dd8ea71)\nFictional decision artifact 036.\n\n### Model B critique (Answer ID: ca_critique_b57d07d859682794474738c9)\nFictional decision artifact 037.\n\n### Model C critique (Answer ID: ca_critique_929e36e04829be9822ceddd9)\nFictional decision artifact 038.\n\nNow provide your claim-audit-aware revised answer.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 039." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:94c516d6931c2ed49b1bb6c232b976ba9485c6e2e3affcab7a6f5165bdd01a39", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer (Answer ID: ca_initial_7aa7c02530241beed834c517):\nFictional decision artifact 034.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_1625c335ddfd9096184d4f8c)\nFictional decision artifact 033.\n\n### Model B initial answer (Answer ID: ca_initial_7aa7c02530241beed834c517)\nFictional decision artifact 034.\n\n### Model C initial answer (Answer ID: ca_initial_e079f32b25f027c8193b28f2)\nFictional decision artifact 035.\n\nAnonymized claim audits:\n\n### Model A critique (Answer ID: ca_critique_3da5202b85c8c8b44dd8ea71)\nFictional decision artifact 036.\n\n### Model B critique (Answer ID: ca_critique_b57d07d859682794474738c9)\nFictional decision artifact 037.\n\n### Model C critique (Answer ID: ca_critique_929e36e04829be9822ceddd9)\nFictional decision artifact 038.\n\nNow provide your claim-audit-aware revised answer.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 040." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:be1accab0b033bc49fc04b41745d9ba84106bf32c6a9e84bd84dde31ff205427", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer (Answer ID: ca_initial_e079f32b25f027c8193b28f2):\nFictional decision artifact 035.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_1625c335ddfd9096184d4f8c)\nFictional decision artifact 033.\n\n### Model B initial answer (Answer ID: ca_initial_7aa7c02530241beed834c517)\nFictional decision artifact 034.\n\n### Model C initial answer (Answer ID: ca_initial_e079f32b25f027c8193b28f2)\nFictional decision artifact 035.\n\nAnonymized claim audits:\n\n### Model A critique (Answer ID: ca_critique_3da5202b85c8c8b44dd8ea71)\nFictional decision artifact 036.\n\n### Model B critique (Answer ID: ca_critique_b57d07d859682794474738c9)\nFictional decision artifact 037.\n\n### Model C critique (Answer ID: ca_critique_929e36e04829be9822ceddd9)\nFictional decision artifact 038.\n\nNow provide your claim-audit-aware revised answer.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 041." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:02bda1be5907a6e502ad841eafee3ad205b35015ffd1749557f414f6348f74ab", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 5, + "messages": [ + { + "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", + "role": "system" + }, + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized decision artifacts:\n\n### Model A decision artifact\nFictional decision artifact 039.\n\n### Model B decision artifact\nFictional decision artifact 040.\n\n### Model C decision artifact\nFictional decision artifact 041.\n\nProduce the consolidated decision artifact.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 042." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:5df9018dc5065ede8d143ca207798c7b9213ca24c76e9a0446723912ec0618ff", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 14, + "messages": [ + { + "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Add conflicts[] only when there are two or more positions in tension.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number \u2014 the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_52b860f7867fc83421bd3ee9) \u2014 from Model A\nFictional decision artifact 039.\n\n### Member answer (answer id: ca_revision_7b938c9dc355e2347e03ced7) \u2014 from Model B\nFictional decision artifact 040.\n\n### Member answer (answer id: ca_revision_9b8324a46592779e0bad8aa1) \u2014 from Model C\nFictional decision artifact 041.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"consensus_score\": {\n \"type\": [\n \"number\",\n \"null\"\n ]\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n },\n \"confidence\": {\n \"type\": \"string\",\n \"enum\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"verdict_applies\"\n ]\n}", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 043." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:4e9493f40db3dbe48b57ec3df9f941b2444101d9775166b4d353c6d4b9a1c9bc", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 32, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 044." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:263812a7f4a8a162ead8f9ea5b142a99b1f931b6beb14c988e0519aa1eab5ae0", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 32, + "messages": [ + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour draft:\nFictional decision artifact 044.\n\nProduce one complete, improved final answer.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 045." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:9a6ce2607e0eb30c7477f008399efde19c5aeed1b7fa1c061620db31b2946fa9", + "occurrence_index": 2, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 046." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:f97d7447f59a3a1923b83ecdbd2a256e788aa0306b80e34487c82f33195cc1c8", + "occurrence_index": 2, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 047." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:8e116cb7c77c18cbe04219e375cd95923cb30625aec753fbc4a4a3ca2907dea3", + "occurrence_index": 2, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 048." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:5891ce106d20b45e9e85dbe28bfee81eb758e095ea0f9a627e572cf8da21a79f", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 046.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 046.\n\n### Model B peer answer\nFictional decision artifact 047.\n\n### Model C peer answer\nFictional decision artifact 048.\n\nProduce one complete revised answer to the original prompt.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 049." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:0d7115e520e1bce4daa06cc0a8cf9b25dc5002ee2d60df2dfa00d38ee9cd58c4", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 047.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 046.\n\n### Model B peer answer\nFictional decision artifact 047.\n\n### Model C peer answer\nFictional decision artifact 048.\n\nProduce one complete revised answer to the original prompt.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 050." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:3fa9111eb2e505f72e5b2b80494e9098f47aab7e444d228f956a54834b76e93f", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 048.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 046.\n\n### Model B peer answer\nFictional decision artifact 047.\n\n### Model C peer answer\nFictional decision artifact 048.\n\nProduce one complete revised answer to the original prompt.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 051." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:4a109d156624f266581a7549ea1fc7661a473c4a0db9fabc2614754a2ac6020f", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 10, + "messages": [ + { + "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", + "role": "system" + }, + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized decision artifacts:\n\n### Model A decision artifact\nFictional decision artifact 049.\n\n### Model B decision artifact\nFictional decision artifact 050.\n\n### Model C decision artifact\nFictional decision artifact 051.\n\nProduce the consolidated decision artifact.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 052." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:9a6ce2607e0eb30c7477f008399efde19c5aeed1b7fa1c061620db31b2946fa9", + "occurrence_index": 3, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 053." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:f97d7447f59a3a1923b83ecdbd2a256e788aa0306b80e34487c82f33195cc1c8", + "occurrence_index": 3, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 054." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:8e116cb7c77c18cbe04219e375cd95923cb30625aec753fbc4a4a3ca2907dea3", + "occurrence_index": 3, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 055." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:75b5a421db80038334f7a5b7e022dac2bc798942ae5fbb5d4f06133aa0c7d82b", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_d31b8fc59637ebad6332cb36)\nFictional decision artifact 053.\n\n### Model B initial answer (Answer ID: ca_initial_1d70996d48de96e92feb4cc2)\nFictional decision artifact 054.\n\n### Model C initial answer (Answer ID: ca_initial_87cb4c02a3e0b70592187ed4)\nFictional decision artifact 055.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 056." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:c9f3331e3a16ca7c13b4a7cab3aceceb523f21b2489c71832187a29c96a6e071", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_d31b8fc59637ebad6332cb36)\nFictional decision artifact 053.\n\n### Model B initial answer (Answer ID: ca_initial_1d70996d48de96e92feb4cc2)\nFictional decision artifact 054.\n\n### Model C initial answer (Answer ID: ca_initial_87cb4c02a3e0b70592187ed4)\nFictional decision artifact 055.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 057." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:d5f31c59028e4f637aff0c1339a5798b39dd5749b87ed5417f68a10a18380e26", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 9, + "messages": [ + { + "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", + "role": "system" + }, + { + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized initial panel:\n\n### Model A initial answer (Answer ID: ca_initial_d31b8fc59637ebad6332cb36)\nFictional decision artifact 053.\n\n### Model B initial answer (Answer ID: ca_initial_1d70996d48de96e92feb4cc2)\nFictional decision artifact 054.\n\n### Model C initial answer (Answer ID: ca_initial_87cb4c02a3e0b70592187ed4)\nFictional decision artifact 055.\n\nAudit the panel using the required claim categories.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 058." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:90717487011b129011d0b7db7fe50382d9967055a302cd0e5324ca0d593e490b", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 10, + "messages": [ + { + "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", + "role": "system" + }, + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized decision artifacts:\n\n### Model A decision artifact\nFictional decision artifact 053.\n\n### Model B decision artifact\nFictional decision artifact 054.\n\n### Model C decision artifact\nFictional decision artifact 055.\n\nAnonymized claim audits:\n\n### Model A anonymized audit\nFictional decision artifact 056.\n\n### Model B anonymized audit\nFictional decision artifact 057.\n\n### Model C anonymized audit\nFictional decision artifact 058.\n\nProduce the consolidated decision artifact.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 059." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:64c25574dc75e5cb8785b3accba9028868bff1e40e1f871b2c7dff341d1958a1", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 16, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 060." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:cfabe1fcf9f9aa97f5ce3c6f5fd00c0b58e0d19b6328b0858efe8559d145ca8a", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 16, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-b", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 061." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:66b06f34e69ffb9fa4bfaa79969147bd90b1c424dfd19a950c80c57a953eeb0b", + "occurrence_index": 1, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 16, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-a", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 062." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:368231f59475acad0fc1c41811ff527bbbe2a94f4acde23932c8d21f178f9c3c", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 16, + "messages": [ + { + "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", + "role": "system" + }, + { + "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nAnonymized decision artifacts:\n\n### Model A decision artifact\nFictional decision artifact 060.\n\n### Model B decision artifact\nFictional decision artifact 061.\n\n### Model C decision artifact\nFictional decision artifact 062.\n\nProduce the consolidated decision artifact.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 063." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + }, + { + "request_hash": "sha256:123070814d2b1c5283590f1d4ee4cd72d3cde87a88ac3d5e6cee374f810c612e", + "occurrence_index": 0, + "request": { + "url": "https://fictional.invalid/v1/chat/completions", + "body": { + "max_tokens": 64, + "messages": [ + { + "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", + "role": "user" + } + ], + "model": "fictional-model-c", + "temperature": 0.0 + } + }, + "status": 200, + "response": { + "choices": [ + { + "message": { + "content": "Fictional decision artifact 064." + } + } + ], + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2 + } + } + } + ] +} From a1d12e8d5acbbd679bfa39bb2bb581cfc82dc558 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 21:38:56 -0400 Subject: [PATCH 10/14] docs(evals): document capped live pilot --- CHANGELOG.md | 5 +++++ DOCUMENTATION_INDEX.md | 12 ++++++------ README.md | 25 +++++++++++++------------ SYSTEM_CONTEXT_DIAGRAM.md | 18 ++++++++++++------ docs/PRODUCT_DESIGN_DOCUMENT.md | 8 +++++++- tests/evals/test_eval_cli.py | 28 ++++++++++++++++++++++++++++ tests/evals/test_qa_pack.py | 14 ++++++++++++++ 7 files changed, 85 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66c45e6..7c41ddb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Version-aware cache identity.** Cache keys cover the resolved roster, generation and mode settings, extraction behavior, sanitized endpoint routing, optional source-bundle digest, and cache/protocol/prompt/schema versions. Old incompatible envelopes are safe misses. +- **Capped live evaluation runner.** The H1 lane is paid exploratory only and dry-run is the + default. Paid execution requires `--execute` plus exact USD 10.00 approval, persists each + reservation before allowing one in-flight provider call, and never repeats an interrupted + cell on resume. Its outputs are not decision eligible: the smoke verifies correctness only, + not efficiency or decision quality; the offline/open-book 24-task fixture remains separate. ### Fixed diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index bb9e486..8ef669d 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -15,15 +15,15 @@ the canonical authority spec on top of those. | # | Doc | Path | Purpose | |---|-----|------|---------| -| 1 | **README** (project overview) | [`README.md`](README.md) | Install, BYO-keys, five released-mode CLI/library quickstart, and the source-only unreleased Elite protocol. | -| 2 | **System Context Diagram** | [`SYSTEM_CONTEXT_DIAGRAM.md`](SYSTEM_CONTEXT_DIAGRAM.md) | Mermaid context for the provider highway, five released modes, source-only Elite gated phases, verdict pipeline, phased manifest receipts, and mcp-warden boundary. | +| 1 | **README** (project overview) | [`README.md`](README.md) | Install, BYO-keys, five released-mode CLI/library quickstart, source-only Elite, and the gated paid exploratory eval lane. | +| 2 | **System Context Diagram** | [`SYSTEM_CONTEXT_DIAGRAM.md`](SYSTEM_CONTEXT_DIAGRAM.md) | Mermaid context for the provider highway, released/source modes, verdict pipeline, eval live guard, manifest receipts, and mcp-warden boundary. | | 3 | **Documentation Index** | [`DOCUMENTATION_INDEX.md`](DOCUMENTATION_INDEX.md) | This file. Master map of all docs + source layout. | ## Authority spec | Doc | Path | Purpose | |-----|------|---------| -| **Product Design Document** | [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md) | **Canonical** product spec: five released modes plus implemented-but-unreleased Elite, its fixed three-success phase gate and cost/latency tradeoff, the v1.1 execution-traceable verdict, manifest, architecture, boundaries, and roadmap. **When docs disagree, the PDD wins.** | +| **Product Design Document** | [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md) | **Canonical** product spec: released/source modes, the v1.1 execution-traceable verdict, manifest, paid exploratory evaluation boundary, architecture, and roadmap. **When docs disagree, the PDD wins.** | ## Product plans @@ -67,7 +67,7 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Models | [`src/conclave/models.py`](src/conclave/models.py) | Stable Pydantic contract, including `EliteResult` phase artifacts and backward-compatible `CouncilResult.elite`. | | CLI | [`src/conclave/cli.py`](src/conclave/cli.py) | Five released `conclave ask` modes plus source-only Elite and `providers`; Elite exits 1 unless decision readiness is `ready`, emits full JSON before failure, and rejects streaming. | | Experimental evals | [`src/conclave/evals/`](src/conclave/evals/) | Versioned DSE-708 planning, strict replay, failure-inclusive running, blinding, scoring, and exploratory reports; no product-quality claim. | -| Eval CLI | [`src/conclave/eval_cli.py`](src/conclave/eval_cli.py) | Offline-only `conclave eval plan/run/blind/report`; `run` validates replay artifacts and cannot call providers. | +| Eval CLI | [`src/conclave/eval_cli.py`](src/conclave/eval_cli.py) | Offline `plan/run/blind/report` plus `eval live`; live defaults to dry-run and paid execution requires `--execute` with exact USD 10.00 approval. | | Logging | [`src/conclave/logging.py`](src/conclave/logging.py) | Logger factory; stderr; verbosity via `CONCLAVE_LOG_LEVEL` (default `WARNING`). | ## Tests @@ -82,7 +82,7 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Provider highway tests | [`tests/test_providers.py`](tests/test_providers.py) | `resolve_adapter` (built-in prefixes, per-provider URLs, custom endpoints, unknown-prefix raise), end-to-end `call_model`, and `redact()` (bearer/`sk-`/env-var-value/`x-api-key` scrubbing; pre-redacted provider errors). | | Registry/config tests | [`tests/test_registry_config.py`](tests/test_registry_config.py) | Name resolution, key-presence logic, config merge. | | CLI tests | [`tests/test_cli.py`](tests/test_cli.py) | Typer `CliRunner`: exit-code contract (0 usable result and ready Elite / 1 zero-usable or non-ready Elite / 2 usage error), `--json` payload + exit code, human renderers per mode, `providers` table never prints secrets, aclose lifecycle. | -| Eval tests | [`tests/evals/`](tests/evals/) | Frozen matrix, replay fail-closed behavior, runner denominators, blinding, scoring, reporting, and offline CLI. | +| Eval tests | [`tests/evals/`](tests/evals/) | Frozen matrix, offline replay, live reservation/checkpoint/resume gates, blinding, scoring, reporting, and CLI assertions. | | Transport tests | [`tests/test_transport.py`](tests/test_transport.py) | `post_json` via httpx `MockTransport`: success/error-status/non-JSON fallback, timeout & connect/HTTP errors โ†’ `TransportError` (key never leaks), client reuse/pooling, aclose idempotency. | | Streaming tests | [`tests/test_streaming.py`](tests/test_streaming.py) | Per-adapter SSE via `MockTransport` (openai-compat/anthropic/gemini): incremental chunks + assembled answer == concatenation == buffered result; mid-stream malformed-frame/connection-drop/non-2xx โ†’ error set with partial text preserved (never raises); key redaction in stream errors; buffered `ask()` never opens a stream; `Council.ask_stream` interleaving + terminal `done` shape; CLI `--stream` smoke + exit-code contract + debate rejection; `--stream` + cache one-shot replay. | | Logging tests | [`tests/test_logging.py`](tests/test_logging.py) | `CONCLAVE_LOG_LEVEL` resolution (default `WARNING`, case-insensitive, unknown โ†’ `WARNING`), factory contract, one-shot configuration. | @@ -116,7 +116,7 @@ Run: `pytest` (config in `pyproject.toml`, `asyncio_mode = "auto"`). | Date | Change | |------|--------| -| 2026-07-18 | Added the validated design and TDD plan for a sequential, checkpointed H1 paid-exploratory runner with frozen pricing and a USD 10 hard cap; implementation and provider calls remain pending. | +| 2026-07-18 | Implemented the sequential paid exploratory runner: default dry-run, exact USD 10.00 execute approval, reservation-before-call, one in-flight call, and no-repeat resume. The 24-task fixture remains offline/open-book and is not the paid smoke corpus; the smoke is correctness-only and not decision eligible. | | 2026-07-17 | Added the experimental DSE-708 offline evaluation substrate and `conclave eval` artifact workflow; no live study or quality claim. | | 2026-07-17 | Reframed the product roadmap around empirically proven decision quality: narrowed current claims to execution traceability, identified answer IDs as internal provenance rather than external evidence, gated Elite merge on H0 correctness, and added H1-H4 evidence, buyer, and outcome gates. | | 2026-07-17 | Documented the implemented-but-unreleased Elite Decision Protocol: fixed three-success gates, initial/claim-audit/revision artifacts, existing final verdict, phased receipts, failure semantics, cost/latency tradeoff, and no streaming. | diff --git a/README.md b/README.md index 70754d8..193ce50 100644 --- a/README.md +++ b/README.md @@ -150,21 +150,22 @@ a future release explicitly includes it. of a council defined in your config (see below). The built-in `default` council is all known providers. -### Experimental offline evaluation (DSE-708) +### Experimental evaluation (DSE-708) -The source branch also exposes an H1 evidence harness. It makes no decision-quality claim and -never calls providers: `run` only validates an existing replay artifact against its frozen -manifest. Grader output and the restricted blind map must remain separate. -The open-book 24-task synthetic QA pack and its no-promotion boundary are documented in -[`studies/elite_qa_v1/README.md`](studies/elite_qa_v1/README.md). Its committed answer keys -are test fixtures, not access-controlled grader material, so this pack cannot be used for a -paid or confirmatory study. +Offline `conclave eval run` validates only a frozen replay artifact and never calls a provider. +The live lane is **paid exploratory only**. Dry-run is the default; execution requires both `--execute` and exact +`--approve-spend-usd 10.00`. During execution, one provider call is in flight, a reservation +is persisted before each call, and resume never repeats an interrupted cell. Outputs stay +labelled paid exploratory and not decision eligible. The 24-task fixture remains +offline/open-book and is not the paid smoke corpus. Smoke checks correctness only, not +efficiency or decision quality; no result supports a product-quality or confirmatory claim. ```bash -conclave eval plan tasks.json manifest.json --study-id qa --replicates 2 --seed 19 --max-output-tokens 1200 -conclave eval run manifest.json validated-run.json --replay-artifact recorded-run.json -conclave eval blind validated-run.json grader.json restricted-map.json --seed 23 -conclave eval report manifest.json validated-run.json judgments.json report.json report.md --bootstrap-seed 29 +conclave eval plan path/tasks.json path/manifest.json --study-id qa --replicates 2 --seed 19 --max-output-tokens 1200 +conclave eval run path/manifest.json path/run.json --replay-artifact path/replay.json +conclave eval live path/manifest.json path/tasks.json path/prices.json path/run.json path/checkpoint.json path/receipts.json +conclave eval live path/manifest.json path/tasks.json path/prices.json path/run.json path/checkpoint.json path/receipts.json \ + --execute --approve-spend-usd 10.00 ``` Add `--stream` to render member (and synthesizer) tokens live as they arrive diff --git a/SYSTEM_CONTEXT_DIAGRAM.md b/SYSTEM_CONTEXT_DIAGRAM.md index f237553..a027db4 100644 --- a/SYSTEM_CONTEXT_DIAGRAM.md +++ b/SYSTEM_CONTEXT_DIAGRAM.md @@ -8,6 +8,7 @@ LLM-SDK dependency), how the v1.1 **verdict pipeline** turns the member answers structured, agreement-scored, **execution-traceable** verdict plus a redacted execution manifest, and how the implemented-but-unreleased **Elite Decision Protocol** adds gated claim audit and revision before that verdict. It also shows where **mcp-warden** sits as a dev-time consumer. +The DSE-708 edge includes offline replay plus an opt-in, capped paid exploratory path. > Authority note: behavioral details here are descriptive. The canonical spec is > [`docs/PRODUCT_DESIGN_DOCUMENT.md`](docs/PRODUCT_DESIGN_DOCUMENT.md). @@ -32,8 +33,9 @@ flowchart TB council["Council orchestrator
fan_out ยท synthesize_blocks ยท skip-no-key (council.py)"] modes["Deliberation modes
debate ยท adversarial ยท vote (modes.py + prompts.py)"] elite["Elite (UNRELEASED)
initial โ†’ claim audit โ†’ revision
fixed 3-success gate each phase"] - evalcli["Experimental eval CLI (DSE-708)
plan ยท replay validation ยท blind ยท report
OFFLINE ONLY"] + evalcli["Experimental eval CLI (DSE-708)
offline replay ยท live dry-run
gated paid exploratory execute"] evalartifacts["Versioned eval artifacts
manifest ยท run ยท grader set + separate map
exploratory report"] + liveguard["Live guard
exact USD 10.00 approval ยท reservation before call
one in flight ยท no-repeat resume"] registry["Registry ยท name to model-id
key PRESENCE only, never values (registry.py)"] config["Config loader ยท custom endpoints (config.py)"] models["Result contract ยท CouncilResult v2
answers ยท verdict ยท consensus ยท manifest (models.py)"] @@ -67,8 +69,9 @@ flowchart TB end user -->|"prompt + council + mode"| cli - user -->|"public tasks + offline artifacts"| evalcli + user -->|"public tasks + frozen artifacts"| evalcli evalcli --> evalartifacts + evalcli -.->|"--execute + exact approval"| liveguard user -->|"import"| lib warden -.->|"imports at DEV time only ยท NOT a runtime dep"| lib @@ -85,6 +88,7 @@ flowchart TB council --> elite elite -->|"3+ revisions: existing synthesis + verdict
under 3: stop incomplete"| council council --> provider + liveguard --> provider provider --> adreg adreg --> oai adreg --> anth @@ -127,10 +131,12 @@ flowchart TB - **Two entry points, one core.** The CLI (`cli.py`) and the library API (`from conclave import Council`) are both thin drivers over the same `Council` orchestrator. There is no behavior in the CLI that the library can't reach. -- **The DSE-708 eval lane is experimental and offline.** `conclave eval` freezes manifests, - validates pre-recorded run artifacts, blinds outputs into a grader set plus separate identity - map, and writes exploratory reports. It has no edge to the provider highway: live execution is - fail-closed, and these artifacts support measurement rather than a decision-quality claim. +- **The DSE-708 eval lane is paid exploratory only.** Offline `eval run` still validates + pre-recorded artifacts without provider calls. `eval live` defaults to dry-run; execution + requires `--execute` plus exact USD 10.00 approval. One provider call is in flight, its + reservation is persisted first, and resume skips an interrupted cell rather than repeating + it. The capped smoke checks correctness only, not efficiency or decision quality, and its + outputs remain not decision eligible. - **mcp-warden is dashed and dev-time.** The dotted edge from `mcp-warden` to the library is deliberate: warden imports conclave **only at design/eval time**. conclave is stochastic and must never sit in warden's deterministic runtime decision path. See PDD diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index d57a616..1b8d176 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -410,7 +410,13 @@ council shipped in v1.1** (ยง4a โ€” the wedge). The revised thesis is a **source-grounded, execution-traceable decision record with empirically proven quality**. Current answer IDs identify model outputs, not external evidence; source-auditable language is therefore too broad until source grounding ships. Elite remains -implemented but unreleased on open PR #51, pending final review and merge. +implemented but unreleased in source. + +H1 also includes an opt-in live runner that is **paid exploratory only**. Dry-run is the default; +paid execution requires `--execute` and exact `--approve-spend-usd 10.00`. One provider call is +in flight, its reservation is persisted before each call, and resume never repeats an interrupted +cell. The 24-task fixture remains offline/open-book and is not the paid smoke corpus. +The smoke establishes correctness only, not efficiency or decision quality; its artifacts remain not decision eligible and cannot change a confirmatory gate. The canonical roadmap is [`docs/plans/2026-07-17-decision-quality-roadmap.md`](plans/2026-07-17-decision-quality-roadmap.md): diff --git a/tests/evals/test_eval_cli.py b/tests/evals/test_eval_cli.py index 2361a38..ee00159 100644 --- a/tests/evals/test_eval_cli.py +++ b/tests/evals/test_eval_cli.py @@ -2,6 +2,7 @@ import json from decimal import Decimal +from pathlib import Path import pytest from typer.testing import CliRunner @@ -22,6 +23,7 @@ from tests.evals.test_live_runner import _live_inputs runner = CliRunner() +ROOT = Path(__file__).resolve().parents[2] def _write_json(path, payload) -> None: @@ -210,6 +212,32 @@ async def forbidden_live_runner(**kwargs): assert provider_calls == 0 +def test_live_evaluation_docs_preserve_operator_and_claim_boundaries() -> None: + docs = { + path: " ".join((ROOT / path).read_text(encoding="utf-8").lower().split()) + for path in ( + "README.md", + "SYSTEM_CONTEXT_DIAGRAM.md", + "DOCUMENTATION_INDEX.md", + "docs/PRODUCT_DESIGN_DOCUMENT.md", + "CHANGELOG.md", + ) + } + corpus = " ".join(docs.values()) + + assert all("paid exploratory" in text for text in docs.values()) + assert "paid exploratory only" in corpus + assert "dry-run is the default" in corpus + assert "--execute" in corpus + assert "--approve-spend-usd 10.00" in corpus + assert "one provider call is in flight" in corpus + assert "reservation is persisted before each call" in corpus + assert "resume never repeats an interrupted cell" in corpus + assert "correctness only" in corpus + assert "not efficiency or decision quality" in corpus + assert "not decision eligible" in corpus + + def test_eval_live_rejects_confirmatory_legacy_or_snapshot_drift( tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/evals/test_qa_pack.py b/tests/evals/test_qa_pack.py index 7b49dac..3697475 100644 --- a/tests/evals/test_qa_pack.py +++ b/tests/evals/test_qa_pack.py @@ -105,3 +105,17 @@ def test_qa_protocol_freezes_open_book_boundary_and_holdout_controls() -> None: assert "eight-token" in preregistration.lower() assert "freeze" in preregistration.lower() assert "unsealing" in preregistration.lower() + + product_docs = " ".join( + " ".join((ROOT / path).read_text(encoding="utf-8").lower().split()) + for path in ( + "README.md", + "SYSTEM_CONTEXT_DIAGRAM.md", + "DOCUMENTATION_INDEX.md", + "docs/PRODUCT_DESIGN_DOCUMENT.md", + "CHANGELOG.md", + ) + ) + assert ( + "24-task fixture remains offline/open-book and is not the paid smoke corpus" in product_docs + ) From bf7aa0de028f26104de834a672f722db79b84e5a Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 22:00:56 -0400 Subject: [PATCH 11/14] fix(evals): preserve paid-call ledger invariants --- src/conclave/evals/live.py | 148 +++++++++++++++++----------- src/conclave/evals/runner.py | 82 +++++++++++++-- src/conclave/models.py | 6 +- tests/evals/test_live_checkpoint.py | 60 +++++++++++ tests/evals/test_live_gateway.py | 114 +++++++++++++++++++++ tests/evals/test_live_runner.py | 117 ++++++++++++++++++++++ 6 files changed, 460 insertions(+), 67 deletions(-) diff --git a/src/conclave/evals/live.py b/src/conclave/evals/live.py index 29db457..c2409c8 100644 --- a/src/conclave/evals/live.py +++ b/src/conclave/evals/live.py @@ -47,6 +47,7 @@ "timeout", "transport", "provider_error", + "reconciliation_error", "reservation_breach", "interrupted", ] @@ -114,6 +115,7 @@ class ProviderCallReceipt(EvalModel): usage: TokenUsage | None = None reserved_cost_usd: Decimal = Field(gt=0) charged_cost_usd: Decimal = Field(ge=0) + hard_cap_breached: bool = False cost_basis: ProviderCallCostBasis @@ -176,6 +178,7 @@ class LiveCheckpoint(EvalModel): bindings: CheckpointBindings committed_cost_usd: Decimal = Field(ge=0) + hard_cap_breached: bool = False records: tuple[RunRecord, ...] = () receipts: tuple[ProviderCallReceipt, ...] = () active_cell: ActiveCell | None = None @@ -196,8 +199,8 @@ def validate_integrity(self) -> LiveCheckpoint: receipt_cost = sum((receipt.charged_cost_usd for receipt in self.receipts), Decimal("0")) if self.committed_cost_usd != receipt_cost: raise ValueError("checkpoint committed cost must equal receipt charges") - if self.committed_cost_usd > self.bindings.hard_cap_usd: - raise ValueError("checkpoint committed cost exceeds the hard cap") + if self.hard_cap_breached != (self.committed_cost_usd > self.bindings.hard_cap_usd): + raise ValueError("checkpoint hard-cap-breached evidence is inconsistent") return self def should_execute(self, planned_run_id: str) -> bool: @@ -252,6 +255,7 @@ def create_live_checkpoint( schema_version=EVAL_SCHEMA_VERSION, bindings=bindings, committed_cost_usd=committed_cost_usd, + hard_cap_breached=committed_cost_usd > bindings.hard_cap_usd, records=tuple(records), receipts=tuple(receipts), active_cell=active_cell, @@ -454,6 +458,8 @@ def finish_active_cell(checkpoint: LiveCheckpoint, *, record: RunRecord) -> Live raise CheckpointValidationError("a final record requires an active checkpoint cell") if active.planned_run_id != record.planned_run_id: raise CheckpointValidationError("final record does not match the active checkpoint cell") + if active.pending_call is not None: + raise CheckpointValidationError("cannot finalize a cell with a pending provider call") return create_live_checkpoint( bindings=checkpoint.bindings, records=(*checkpoint.records, record), @@ -527,6 +533,14 @@ def _bounded_error_category(error: str) -> ProviderCallErrorCategory: return _receipt_error_category(error) +def _full_reservation_cost_basis(reservation: CallReservation) -> ProviderCallCostBasis: + return ProviderCallCostBasis( + source="full_reservation", + input_ceiling_usd_per_million_tokens=(reservation.input_ceiling_usd_per_million_tokens), + output_ceiling_usd_per_million_tokens=(reservation.output_ceiling_usd_per_million_tokens), + ) + + class LiveProviderClient: """Guard all eval provider calls with serialization and exact cost accounting.""" @@ -647,75 +661,95 @@ async def call(self, call: StageCall) -> ModelAnswer: error=category, ) - category: ProviderCallErrorCategory | None = None - if answer.error is not None: - category = _bounded_error_category(answer.error) - answer = answer.model_copy(update={"error": category}) - - usage = answer.usage - if usage is None: - charged_cost = reservation.reserved_cost_usd - cost_basis = ProviderCallCostBasis( - source="full_reservation", - input_ceiling_usd_per_million_tokens=( - reservation.input_ceiling_usd_per_million_tokens - ), - output_ceiling_usd_per_million_tokens=( - reservation.output_ceiling_usd_per_million_tokens + reconciliation_failed = False + breached = False + try: + category: ProviderCallErrorCategory | None = None + if answer.error is not None: + category = _bounded_error_category(answer.error) + answer = answer.model_copy(update={"error": category}) + + usage = answer.usage + if usage is None: + charged_cost = reservation.reserved_cost_usd + cost_basis = _full_reservation_cost_basis(reservation) + else: + usage = TokenUsage.model_validate( + { + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + } + ) + charged_cost = _reported_usage_cost(price, usage) + cost_basis = ProviderCallCostBasis( + source="reported_usage", + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + input_ceiling_usd_per_million_tokens=( + reservation.input_ceiling_usd_per_million_tokens + ), + output_ceiling_usd_per_million_tokens=( + reservation.output_ceiling_usd_per_million_tokens + ), + ) + breached = self._usage_breaches(usage, reservation) + + if breached: + category = "reservation_breach" + answer = answer.model_copy(update={"answer": None, "error": category}) + + receipt = ProviderCallReceipt( + stage=call.stage, + provider_id=call.provider_id, + model_id=call.model_id, + model_revision=call.model_revision, + max_output_tokens=call.max_output_tokens, + outcome="failed" if category is not None else "success", + error_category=category, + usage=usage, + reserved_cost_usd=reservation.reserved_cost_usd, + charged_cost_usd=charged_cost, + hard_cap_breached=( + self._committed_cost_usd + charged_cost > self._hard_cap_usd ), + cost_basis=cost_basis, ) - breached = False - else: - charged_cost = _reported_usage_cost(price, usage) - cost_basis = ProviderCallCostBasis( - source="reported_usage", - prompt_tokens=usage.prompt_tokens, - completion_tokens=usage.completion_tokens, - input_ceiling_usd_per_million_tokens=( - reservation.input_ceiling_usd_per_million_tokens - ), - output_ceiling_usd_per_million_tokens=( - reservation.output_ceiling_usd_per_million_tokens - ), + except Exception: # noqa: BLE001 -- fail closed with a bounded paid-call receipt + reconciliation_failed = True + category = "reconciliation_error" + answer = ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + error=category, ) - breached = self._usage_breaches(usage, reservation) - - if breached: - category = "reservation_breach" - answer = answer.model_copy(update={"answer": None, "error": category}) charged_cost = reservation.reserved_cost_usd - cost_basis = ProviderCallCostBasis( - source="full_reservation", - input_ceiling_usd_per_million_tokens=( - reservation.input_ceiling_usd_per_million_tokens - ), - output_ceiling_usd_per_million_tokens=( - reservation.output_ceiling_usd_per_million_tokens + receipt = ProviderCallReceipt( + stage=call.stage, + provider_id=call.provider_id, + model_id=call.model_id, + model_revision=call.model_revision, + max_output_tokens=call.max_output_tokens, + outcome="failed", + error_category=category, + reserved_cost_usd=reservation.reserved_cost_usd, + charged_cost_usd=charged_cost, + hard_cap_breached=( + self._committed_cost_usd + charged_cost > self._hard_cap_usd ), + cost_basis=_full_reservation_cost_basis(reservation), ) - - receipt = ProviderCallReceipt( - stage=call.stage, - provider_id=call.provider_id, - model_id=call.model_id, - model_revision=call.model_revision, - max_output_tokens=call.max_output_tokens, - outcome="failed" if category is not None else "success", - error_category=category, - usage=usage, - reserved_cost_usd=reservation.reserved_cost_usd, - charged_cost_usd=charged_cost, - cost_basis=cost_basis, - ) self._committed_cost_usd += charged_cost self._receipts.append(receipt) self._pending_call = None - if breached: + if breached or reconciliation_failed: self._stopped = True await self._persist() if breached: raise ReservationBreachError("provider usage breached the call reservation") + if reconciliation_failed: + raise GatewayStoppedError("provider response reconciliation failed") return answer diff --git a/src/conclave/evals/runner.py b/src/conclave/evals/runner.py index 34f57de..d3a7f23 100644 --- a/src/conclave/evals/runner.py +++ b/src/conclave/evals/runner.py @@ -3,9 +3,11 @@ from __future__ import annotations import asyncio +import os import time from collections import Counter -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager from decimal import Decimal from pathlib import Path @@ -47,6 +49,11 @@ ) from .pricing import PriceBook, validate_price_book +try: + import fcntl as _fcntl +except ImportError: # pragma: no cover - exercised by the fail-closed seam + _fcntl = None + LIVE_HARD_CAP_USD = Decimal("10.00") ProtocolExecutor = Callable[ @@ -59,6 +66,43 @@ class RunValidationError(ValueError): """Execution inputs or outputs do not exactly match the frozen manifest.""" +@contextmanager +def _checkpoint_lifecycle_lease(checkpoint_path: Path) -> Iterator[None]: + """Hold one nonblocking process lease for the checkpoint's full lifecycle.""" + + if _fcntl is None: + raise CheckpointValidationError( + "checkpoint filesystem leases are unsupported on this platform" + ) + + lock_path = checkpoint_path.with_name(f"{checkpoint_path.name}.lock") + descriptor: int | None = None + acquired = False + try: + flags = os.O_CREAT | os.O_RDWR + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + descriptor = os.open(lock_path, flags, 0o600) + _fcntl.flock(descriptor, _fcntl.LOCK_EX | _fcntl.LOCK_NB) + acquired = True + except (AttributeError, OSError): + if descriptor is not None: + os.close(descriptor) + raise CheckpointValidationError( + "checkpoint filesystem lease is unavailable or unsupported" + ) from None + + try: + yield + finally: + if descriptor is not None: + try: + if acquired: + _fcntl.flock(descriptor, _fcntl.LOCK_UN) + finally: + os.close(descriptor) + + def validate_run_records( manifest: StudyManifest, records: Sequence[RunRecord] ) -> tuple[RunRecord, ...]: @@ -415,6 +459,28 @@ async def run_live_study( task_by_id, roster_by_id = _validate_live_inputs(manifest, tasks, price_book) path = Path(checkpoint_path) + with _checkpoint_lifecycle_lease(path): + return await _run_live_study_with_lease( + manifest=manifest, + price_book=price_book, + path=path, + task_by_id=task_by_id, + roster_by_id=roster_by_id, + call_model_func=call_model_func, + ) + + +async def _run_live_study_with_lease( + *, + manifest: StudyManifest, + price_book: PriceBook, + path: Path, + task_by_id: Mapping[str, PublicTask], + roster_by_id: Mapping[str, tuple], + call_model_func: CallModel, +) -> StudyRun: + """Run one validated live study while its process lease is held.""" + checkpoint = _load_or_create_live_checkpoint( path, manifest=manifest, @@ -486,13 +552,15 @@ def persist( write_live_checkpoint(path, checkpoint) break except (ReservationBreachError, GatewayStoppedError) as exc: - category = ( - "reservation_breach" - if isinstance(exc, ReservationBreachError) - else "gateway_stopped" - ) + receipts = _cell_receipts(checkpoint, receipt_start_index=receipt_start_index) + if isinstance(exc, ReservationBreachError): + category = "reservation_breach" + elif receipts and receipts[-1].error_category == "reconciliation_error": + category = "reconciliation_error" + else: + category = "gateway_stopped" execution = _aggregate_live_execution( - _cell_receipts(checkpoint, receipt_start_index=receipt_start_index), + receipts, latency_ms=(time.perf_counter() - started) * 1000, error_category=category, deviation_codes=(category,), diff --git a/src/conclave/models.py b/src/conclave/models.py index 5d0efd6..826a039 100644 --- a/src/conclave/models.py +++ b/src/conclave/models.py @@ -110,9 +110,9 @@ def _default_prompt_version() -> str: class TokenUsage(BaseModel): """Token accounting for a single model call.""" - prompt_tokens: int = 0 - completion_tokens: int = 0 - total_tokens: int = 0 + prompt_tokens: int = Field(default=0, ge=0) + completion_tokens: int = Field(default=0, ge=0) + total_tokens: int = Field(default=0, ge=0) class ModelAnswer(BaseModel): diff --git a/tests/evals/test_live_checkpoint.py b/tests/evals/test_live_checkpoint.py index 35b50fe..3e51532 100644 --- a/tests/evals/test_live_checkpoint.py +++ b/tests/evals/test_live_checkpoint.py @@ -18,7 +18,9 @@ PendingCall, ProviderCallCostBasis, ProviderCallReceipt, + ReservationBreachError, create_live_checkpoint, + finish_active_cell, load_live_checkpoint, recover_interrupted_checkpoint, start_active_cell, @@ -286,6 +288,43 @@ async def resumed_provider(name, model_id, messages, **kwargs): assert resumed.receipts[-1].cost_basis.source == "full_reservation" +@pytest.mark.asyncio +async def test_checkpoint_persists_honest_hard_cap_breach_evidence(tmp_path) -> None: + hard_cap = Decimal("0.000104") + bindings = _bindings(hard_cap_usd=hard_cap) + path = tmp_path / "hard-cap-breach.json" + state = start_active_cell(create_live_checkpoint(bindings=bindings), planned_run_id=RUN_A) + + def persist(pending, receipts) -> None: + nonlocal state + state = update_live_checkpoint(state, pending_call=pending, receipts=receipts) + write_live_checkpoint(path, state) + + async def provider(name, model_id, messages, **kwargs): + del messages, kwargs + return ModelAnswer( + name=name, + model_id=model_id, + answer="over-cap fixture", + usage=TokenUsage(prompt_tokens=200, completion_tokens=6, total_tokens=206), + ) + + client = LiveProviderClient( + price_book=_price_book(), + hard_cap_usd=hard_cap, + checkpoint=persist, + call_model_func=provider, + ) + with pytest.raises(ReservationBreachError): + await client.call(_stage_call()) + + persisted = load_live_checkpoint(path, expected_bindings=bindings) + assert persisted.committed_cost_usd == Decimal("0.000206") + assert persisted.committed_cost_usd > hard_cap + assert persisted.hard_cap_breached is True + assert persisted.receipts[-1].hard_cap_breached is True + + def test_resume_preserves_completed_records_and_call_receipts() -> None: completed_record = RunRecord( planned_run_id=RUN_A, @@ -321,6 +360,27 @@ def test_resume_preserves_completed_records_and_call_receipts() -> None: ) +def test_finish_active_cell_refuses_to_clear_pending_paid_call() -> None: + pending = _pending_call() + checkpoint = create_live_checkpoint( + bindings=_bindings(), + active_cell=ActiveCell( + planned_run_id=RUN_A, + receipt_start_index=0, + pending_call=pending, + ), + ) + record = RunRecord( + planned_run_id=RUN_A, + outcome="failed", + error_category="protocol_error", + cost_receipt_complete=True, + ) + + with pytest.raises(CheckpointValidationError, match="pending"): + finish_active_cell(checkpoint, record=record) + + def test_corrupt_or_tampered_checkpoint_fails_closed(tmp_path) -> None: path = tmp_path / "checkpoint.json" checkpoint = create_live_checkpoint(bindings=_bindings()) diff --git a/tests/evals/test_live_gateway.py b/tests/evals/test_live_gateway.py index 1d7e040..f5e8d76 100644 --- a/tests/evals/test_live_gateway.py +++ b/tests/evals/test_live_gateway.py @@ -6,6 +6,7 @@ import pytest from pydantic import ValidationError +import conclave.evals.live as live_module from conclave.evals.live import ( BudgetExceededError, GatewayStoppedError, @@ -220,6 +221,119 @@ async def fake_call_model(name, model_id, messages, **kwargs): assert provider_calls == 1 +@pytest.mark.asyncio +async def test_gateway_records_actual_cost_when_breach_exceeds_hard_cap() -> None: + hard_cap = Decimal("0.000086") + + async def fake_call_model(name, model_id, messages, **kwargs): + del messages, kwargs + return ModelAnswer( + name=name, + model_id=model_id, + answer="provider exceeded the reservation and hard cap", + usage=TokenUsage(prompt_tokens=100, completion_tokens=6, total_tokens=106), + ) + + client = LiveProviderClient( + price_book=_price_book(input_rate="1", output_rate="1"), + hard_cap_usd=hard_cap, + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + + with pytest.raises(ReservationBreachError, match="reservation"): + await client.call(_stage_call(cap=5)) + + receipt = client.receipts[-1] + assert receipt.reserved_cost_usd == hard_cap + assert receipt.charged_cost_usd == Decimal("0.000106") + assert receipt.charged_cost_usd > receipt.reserved_cost_usd + assert receipt.charged_cost_usd > hard_cap + assert receipt.cost_basis.source == "reported_usage" + assert receipt.error_category == "reservation_breach" + assert receipt.hard_cap_breached is True + assert client.committed_cost_usd == receipt.charged_cost_usd + + +def test_token_usage_rejects_negative_counts() -> None: + with pytest.raises(ValidationError): + TokenUsage(prompt_tokens=-1, completion_tokens=0, total_tokens=0) + + +@pytest.mark.parametrize( + "usage", + ( + TokenUsage.model_construct(prompt_tokens=-1, completion_tokens=1, total_tokens=0), + TokenUsage.model_construct(prompt_tokens="malformed", completion_tokens=1, total_tokens=1), + ), +) +@pytest.mark.asyncio +async def test_gateway_persists_reconciliation_error_for_malformed_usage(usage) -> None: + checkpoints: list[tuple[object | None, tuple[ProviderCallReceipt, ...]]] = [] + + async def fake_call_model(name, model_id, messages, **kwargs): + del messages, kwargs + return ModelAnswer.model_construct( + name=name, + model_id=model_id, + answer="malformed usage fixture", + usage=usage, + error=None, + ) + + client = LiveProviderClient( + price_book=_price_book(input_rate="1", output_rate="1"), + hard_cap_usd=Decimal("1"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: checkpoints.append((pending, receipts)), + ) + + with pytest.raises(GatewayStoppedError, match="reconciliation"): + await client.call(_stage_call()) + + receipt = client.receipts[-1] + assert receipt.error_category == "reconciliation_error" + assert receipt.outcome == "failed" + assert receipt.usage is None + assert receipt.charged_cost_usd == receipt.reserved_cost_usd + assert receipt.cost_basis.source == "full_reservation" + assert client.pending_call is None + assert client.stopped is True + assert checkpoints[-1] == (None, client.receipts) + + +@pytest.mark.asyncio +async def test_gateway_persists_reconciliation_error_when_costing_raises(monkeypatch) -> None: + async def fake_call_model(name, model_id, messages, **kwargs): + del messages, kwargs + return ModelAnswer( + name=name, + model_id=model_id, + answer="valid provider response", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + def broken_costing(price, usage): + del price, usage + raise ArithmeticError("unbounded internal fixture detail") + + monkeypatch.setattr(live_module, "_reported_usage_cost", broken_costing) + client = LiveProviderClient( + price_book=_price_book(input_rate="1", output_rate="1"), + hard_cap_usd=Decimal("1"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + + with pytest.raises(GatewayStoppedError, match="reconciliation"): + await client.call(_stage_call()) + + receipt = client.receipts[-1] + assert receipt.error_category == "reconciliation_error" + assert receipt.charged_cost_usd == receipt.reserved_cost_usd + assert "unbounded internal fixture detail" not in receipt.model_dump_json() + + @pytest.mark.asyncio async def test_gateway_receipt_contains_bounded_error_category_not_raw_exception() -> None: secret = "sk-live-secret-value" diff --git a/tests/evals/test_live_runner.py b/tests/evals/test_live_runner.py index 956ad65..20888be 100644 --- a/tests/evals/test_live_runner.py +++ b/tests/evals/test_live_runner.py @@ -1,12 +1,15 @@ from __future__ import annotations +import asyncio import json +import multiprocessing from collections import Counter from decimal import Decimal import pytest import conclave.evals.dataset as dataset_module +import conclave.evals.runner as runner_module from conclave.evals.live import ( CheckpointValidationError, build_checkpoint_bindings, @@ -149,6 +152,54 @@ async def _successful_provider(name, model_id, messages, **kwargs): ) +def _run_live_study_process( + manifest, + tasks, + price_book, + checkpoint_path, + start_event, + lease_held_event, + release_event, + results, +) -> None: + calls = 0 + + async def provider(name, model_id, messages, **kwargs): + nonlocal calls + del messages, kwargs + calls += 1 + if calls == 1: + lease_held_event.set() + if not release_event.wait(timeout=10): + raise RuntimeError("fixture lease release timed out") + return ModelAnswer( + name=name, + model_id=model_id, + answer=f"fixture answer from {name}", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + if not start_event.wait(timeout=10): + results.put(("start_timeout", calls)) + return + try: + asyncio.run( + run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=checkpoint_path, + call_model_func=provider, + ) + ) + except CheckpointValidationError: + results.put(("lease_rejected", calls)) + except BaseException as exc: # noqa: BLE001 - child reports bounded test status + results.put((f"unexpected:{type(exc).__name__}", calls)) + else: + results.put(("completed", calls)) + + @pytest.mark.asyncio async def test_live_runner_requires_paid_exploratory_frozen_manifest(tmp_path) -> None: provider_calls = 0 @@ -273,6 +324,72 @@ async def recording_provider(name, model_id, messages, **kwargs): ) +def test_live_runner_allows_only_one_process_per_checkpoint(tmp_path) -> None: + context = multiprocessing.get_context("fork") + tasks, manifest, price_book = _live_inputs() + checkpoint_path = tmp_path / "leased-checkpoint.json" + start_event = context.Event() + lease_held_event = context.Event() + release_event = context.Event() + results = context.Queue() + args = ( + manifest, + tasks, + price_book, + checkpoint_path, + start_event, + lease_held_event, + release_event, + results, + ) + processes = [context.Process(target=_run_live_study_process, args=args) for _ in range(2)] + for process in processes: + process.start() + + try: + start_event.set() + assert lease_held_event.wait(timeout=10) + rejected = results.get(timeout=10) + assert rejected == ("lease_rejected", 0) + release_event.set() + completed = results.get(timeout=20) + assert completed[0] == "completed" + assert completed[1] > 0 + finally: + release_event.set() + for process in processes: + process.join(timeout=10) + if process.is_alive(): + process.terminate() + process.join(timeout=5) + assert all(process.exitcode == 0 for process in processes) + + +@pytest.mark.asyncio +async def test_live_runner_fails_closed_when_filesystem_leases_are_unsupported( + tmp_path, monkeypatch +) -> None: + tasks, manifest, price_book = _live_inputs() + provider_calls = 0 + + async def forbidden_provider(*args, **kwargs): + nonlocal provider_calls + del args, kwargs + provider_calls += 1 + raise AssertionError("unsupported lease platform must fail before provider access") + + monkeypatch.setattr(runner_module, "_fcntl", None, raising=False) + with pytest.raises(CheckpointValidationError, match="unsupported"): + await run_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + checkpoint_path=tmp_path / "unsupported-lease.json", + call_model_func=forbidden_provider, + ) + assert provider_calls == 0 + + @pytest.mark.asyncio async def test_budget_exhaustion_makes_no_call_and_marks_all_remaining_cells_incomplete( tmp_path, From 1577c8be6af859bead047ae890de0b15c585dc1e Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 22:22:29 -0400 Subject: [PATCH 12/14] fix(evals): enforce live Elite verdict contract --- ...-07-18-h1-live-evaluation-runner-design.md | 16 +- src/conclave/evals/live.py | 31 +- src/conclave/evals/live_protocols.py | 106 +++++-- src/conclave/verdict_synthesis.py | 10 +- tests/evals/test_live_estimate.py | 18 ++ tests/evals/test_live_gateway.py | 41 ++- tests/evals/test_live_protocols.py | 167 +++++++++- tests/evals/test_live_replay.py | 38 ++- tests/evals/test_live_runner.py | 51 +++- tests/fixtures/evals/live_smoke/manifest.json | 62 ++-- tests/fixtures/evals/live_smoke/replay.json | 288 +++++++++--------- tests/test_verdict_synthesis.py | 18 ++ 12 files changed, 622 insertions(+), 224 deletions(-) diff --git a/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md b/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md index 20088e7..52ab036 100644 --- a/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md +++ b/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md @@ -89,11 +89,17 @@ per roster so `elite_full` exercises its implemented three-responder contract. Elite prompt builders; the first member then produces the same synthesis and structured verdict stages required by the current Elite protocol. -A frozen allocation table divides each cell ceiling across its stages; integer remainder is -assigned to the final graded-output stage. The allocator rejects a cell ceiling too small to -give every call the minimum useful stage cap. Failed responder gates stop later stages and -produce a failure-inclusive cell record. The graded output is the condition's final decision -artifact, never an internal critique. +A frozen allocation table divides each cell ceiling across its maximum call graph, including +Elite's optional verdict repair. Version `live_stage_minimum_caps_v1` reserves 256 output tokens +for `initial`, `draft`, and `critique`; 384 for `self_revision` and `revision`; 512 for +`synthesis`; and 768 for each of `verdict` and `verdict_repair`. These conservative floors give +short reasoning stages useful space, give integration stages more room, and fit the current +multi-position structured verdict on either attempt. A three-member Elite cell therefore +requires at least 4,736 output tokens before any call. Extra tokens are distributed +deterministically and exactly; integer remainder goes to the normal graded-output stage while +the optional repair retains its full floor. Failed responder gates stop later stages and produce +a failure-inclusive cell record. The graded output is the condition's canonical final decision +artifact, never an internal critique or unvalidated extractor text. ## Data flow diff --git a/src/conclave/evals/live.py b/src/conclave/evals/live.py index c2409c8..3c43ac8 100644 --- a/src/conclave/evals/live.py +++ b/src/conclave/evals/live.py @@ -651,6 +651,7 @@ async def call(self, call: StageCall) -> ModelAnswer: temperature=self._temperature, timeout=self._timeout, max_output_tokens=call.max_output_tokens, + output_contract=call.output_contract, ) except Exception as exc: # noqa: BLE001 -- injected seam may raise raw_error = f"{type(exc).__name__}: {exc}" @@ -763,6 +764,14 @@ def __init__(self, price_book: PriceBook) -> None: async def call(self, call: StageCall) -> ModelAnswer: price = _resolve_call_price(self._price_book, call) self.reservations.append(_reserve_stage_call(call, price)) + if call.stage == "verdict": + return ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + answer="invalid-first-verdict-for-max-graph-estimate", + ) + if call.stage == "verdict_repair": + raise _EstimateMaxGraphComplete return ModelAnswer( name=call.provider_id, model_id=call.model_id, @@ -770,6 +779,10 @@ async def call(self, call: StageCall) -> ModelAnswer: ) +class _EstimateMaxGraphComplete(Exception): + """Internal signal emitted after the optional repair reservation is recorded.""" + + async def estimate_live_study( *, manifest: StudyManifest, @@ -791,13 +804,17 @@ async def estimate_live_study( for planned_run in manifest.planned_runs: reservation_start = len(client.reservations) - await execute_live_condition( - planned_run.condition_id, - task=task_by_id[planned_run.task_id], - roster=roster_by_id[planned_run.roster_id], - cell_ceiling=planned_run.max_output_tokens, - client=client, - ) + try: + await execute_live_condition( + planned_run.condition_id, + task=task_by_id[planned_run.task_id], + roster=roster_by_id[planned_run.roster_id], + cell_ceiling=planned_run.max_output_tokens, + client=client, + ) + except _EstimateMaxGraphComplete: + if planned_run.condition_id != "elite_full": # pragma: no cover - invariant + raise cell_upper_bound = sum( ( reservation.reserved_cost_usd diff --git a/src/conclave/evals/live_protocols.py b/src/conclave/evals/live_protocols.py index 1697068..8294658 100644 --- a/src/conclave/evals/live_protocols.py +++ b/src/conclave/evals/live_protocols.py @@ -10,10 +10,11 @@ from pydantic import Field from conclave import prompts +from conclave.adapters.base import OutputContract from conclave.council import _SYNTH_SYSTEM from conclave.models import ELITE_MIN_RESPONDERS, ModelAnswer, derive_phase_answer_id from conclave.verdict import VERDICT_EXTRACTION_PROMPT_VERSION -from conclave.verdict_synthesis import _build_messages as _verdict_messages +from conclave.verdict_synthesis import extract_verdict from .models import ( EVAL_CONDITION_IDS, @@ -31,6 +32,7 @@ "revision", "synthesis", "verdict", + "verdict_repair", ] @@ -51,6 +53,7 @@ class StageCall(EvalModel): messages: tuple[ChatMessage, ...] = Field(min_length=1) max_output_tokens: int = Field(gt=0) upstream_output_token_ceilings: tuple[int, ...] = () + output_contract: OutputContract | None = None class SequentialStageClient(Protocol): @@ -92,10 +95,31 @@ class LiveProtocolSpec: ("revision", "all"), ("synthesis", "lead"), ("verdict", "lead"), + ("verdict_repair", "lead"), ), } ) +# These are minimum *output* caps, not expected response lengths. They keep a +# frozen live manifest from authorizing calls that cannot produce a useful +# artifact even in principle. Member answers/audits need room for a short +# reasoned response; revisions need room to integrate peers; synthesis needs a +# complete decision artifact; and both structured-verdict attempts need enough +# room for the current multi-position CouncilVerdict JSON shape. +LIVE_STAGE_MINIMUM_CAPS_VERSION = "live_stage_minimum_caps_v1" +LIVE_STAGE_MINIMUM_CAPS: Mapping[StageName, int] = MappingProxyType( + { + "initial": 256, + "draft": 256, + "self_revision": 384, + "critique": 256, + "revision": 384, + "synthesis": 512, + "verdict": 768, + "verdict_repair": 768, + } +) + LIVE_PROMPT_VERSIONS: Mapping[ConditionId, str] = MappingProxyType( { "single_frontier": "live_single_frontier_prompt_v1", @@ -131,17 +155,40 @@ def stage_call_sequence( def allocate_stage_caps( condition_id: ConditionId, *, roster_size: int, cell_ceiling: int ) -> tuple[int, ...]: - """Allocate positive equal caps, assigning integer remainder to graded output.""" + """Allocate versioned useful minima plus deterministic exact headroom.""" if isinstance(cell_ceiling, bool) or not isinstance(cell_ceiling, int): raise TypeError("cell_ceiling must be an integer") - call_count = len(stage_call_sequence(condition_id, roster_size=roster_size)) - if cell_ceiling < call_count: + sequence = stage_call_sequence(condition_id, roster_size=roster_size) + call_count = len(sequence) + minimum_caps = [LIVE_STAGE_MINIMUM_CAPS[stage] for stage, _index in sequence] + minimum = sum(minimum_caps) + if cell_ceiling < minimum: raise ValueError( - f"cell ceiling {cell_ceiling} is too small for {call_count} positive stage caps" + f"cell ceiling {cell_ceiling} is too small for {condition_id}; " + f"{LIVE_STAGE_MINIMUM_CAPS_VERSION} requires at least {minimum} tokens" ) - per_call, remainder = divmod(cell_ceiling, call_count) - return (*([per_call] * (call_count - 1)), per_call + remainder) + per_call_extra, remainder = divmod(cell_ceiling - minimum, call_count) + caps = [cap + per_call_extra for cap in minimum_caps] + graded_index = ( + next(index for index, (stage, _member) in enumerate(sequence) if stage == "verdict") + if condition_id == "elite_full" + else call_count - 1 + ) + caps[graded_index] += remainder + return tuple(caps) + + +def minimum_cell_ceiling(condition_id: ConditionId, *, roster_size: int) -> int: + """Return the versioned minimum useful token ceiling for one max call graph.""" + + return sum( + LIVE_STAGE_MINIMUM_CAPS[stage] + for stage, _member_index in stage_call_sequence( + condition_id, + roster_size=roster_size, + ) + ) def _public_prompt(task: PublicTask) -> str: @@ -239,6 +286,7 @@ async def call( messages: tuple[ChatMessage, ...], *, upstream: Sequence[int] = (), + output_contract: OutputContract | None = None, ) -> ModelAnswer: member = self.roster[member_index] return await self.client.call( @@ -250,6 +298,7 @@ async def call( messages=messages, max_output_tokens=self._caps[(stage, member_index)], upstream_output_token_ceilings=tuple(upstream), + output_contract=output_contract, ) ) @@ -447,16 +496,39 @@ async def execute_elite_full(run: _Execution, public_prompt: str) -> str: upstream=run.caps("revision", [index for index, _answer in revisions]), ) _required(synthesis, "synthesis") - verdict = await run.call( - "verdict", - 0, - tuple( - ChatMessage.model_validate(message) - for message in _verdict_messages(public_prompt, revision_answers) - ), - upstream=run.caps("revision", [index for index, _answer in revisions]), + verdict_attempt = 0 + + async def guarded_verdict_call(name, model_id, messages, **kwargs): # noqa: ANN001 + nonlocal verdict_attempt + lead = run.roster[0] + if (name, model_id) != (lead.provider_id, lead.model_id): + raise ValueError("verdict extraction drifted from the frozen lead model") + verdict_attempt += 1 + if verdict_attempt > 2: + raise ValueError("verdict extraction exceeded one repair attempt") + stage: StageName = "verdict" if verdict_attempt == 1 else "verdict_repair" + return await run.call( + stage, + 0, + tuple(ChatMessage.model_validate(message) for message in messages), + upstream=run.caps("revision", [index for index, _answer in revisions]), + output_contract=kwargs.get("output_contract"), + ) + + lead = run.roster[0] + verdict_result = await extract_verdict( + public_prompt, + revision_answers, + synthesizer_name=lead.provider_id, + synthesizer_model_id=lead.model_id, + temperature=0.0, + protocol_version="elite_full_live_v2", + call_model_func=guarded_verdict_call, ) - return _required(verdict, "verdict") + if verdict_result.verdict is None: + reason = verdict_result.verdict_absent_reason or "unknown verdict failure" + raise ValueError(f"verdict stage did not produce a canonical CouncilVerdict: {reason}") + return verdict_result.verdict.model_dump_json() _EXECUTORS: Mapping[ConditionId, Callable[[_Execution, str], Awaitable[str]]] = MappingProxyType( @@ -474,7 +546,7 @@ async def execute_elite_full(run: _Execution, public_prompt: str) -> str: { condition_id: LiveProtocolSpec( condition_id=condition_id, - protocol_version=f"{condition_id}_live_v1", + protocol_version=f"{condition_id}_live_v2", prompt_version=LIVE_PROMPT_VERSIONS[condition_id], executor=_EXECUTORS[condition_id], ) diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index 608f1ef..362b4da 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -444,6 +444,7 @@ async def extract_verdict( temperature: float = 0.7, timeout: float = 120.0, protocol_version: str | None = None, + call_model_func=None, # noqa: ANN001 -- injectable async seam for guarded callers ) -> VerdictSynthesisResult: """Extract a structured, auditable verdict from a council's member answers. @@ -481,6 +482,10 @@ async def extract_verdict( ``"anthropic/claude-sonnet-4"``). Recorded as provenance. config: Optional pre-resolved :class:`conclave.config.ConclaveConfig` threaded through to ``call_model`` (custom endpoints / no re-read). + call_model_func: Optional async call seam matching :func:`call_model`. + ``None`` preserves the normal module-level provider path. Guarded + eval runners inject their reservation-aware gateway here so both the + initial extraction and optional repair remain paid-call protected. Returns: A :class:`VerdictSynthesisResult`. On success ``verdict`` is populated and @@ -528,7 +533,8 @@ async def extract_verdict( schema_name="VerdictExtraction", strict=True, ) - answer = await call_model( + model_caller = call_model if call_model_func is None else call_model_func + answer = await model_caller( synthesizer_name, synthesizer_model_id, messages, @@ -564,7 +570,7 @@ async def extract_verdict( ), } ] - retry = await call_model( + retry = await model_caller( synthesizer_name, synthesizer_model_id, repair_messages, diff --git a/tests/evals/test_live_estimate.py b/tests/evals/test_live_estimate.py index a1b9a4f..ddb45b8 100644 --- a/tests/evals/test_live_estimate.py +++ b/tests/evals/test_live_estimate.py @@ -98,6 +98,24 @@ def recording_reserve(*args, **kwargs): estimate.maximum_call_count = 0 +@pytest.mark.asyncio +async def test_dry_run_prices_the_optional_elite_verdict_repair_max_graph() -> None: + tasks, manifest, price_book = _live_inputs() + + estimate = await estimate_live_study( + manifest=manifest, + tasks=tasks, + price_book=price_book, + ) + + elite_max_graph = stage_call_sequence("elite_full", roster_size=3) + assert elite_max_graph[-2:] == (("verdict", 0), ("verdict_repair", 0)) + expected_calls = sum( + len(stage_call_sequence(run.condition_id, roster_size=3)) for run in manifest.planned_runs + ) + assert estimate.maximum_call_count == expected_calls + + @pytest.mark.asyncio async def test_dry_run_breaks_down_upper_bound_by_roster_and_condition() -> None: tasks, manifest, price_book = _live_inputs() diff --git a/tests/evals/test_live_gateway.py b/tests/evals/test_live_gateway.py index f5e8d76..26c25fa 100644 --- a/tests/evals/test_live_gateway.py +++ b/tests/evals/test_live_gateway.py @@ -7,6 +7,7 @@ from pydantic import ValidationError import conclave.evals.live as live_module +from conclave.adapters.base import OutputContract from conclave.evals.live import ( BudgetExceededError, GatewayStoppedError, @@ -40,7 +41,12 @@ def _price_book( ) -def _stage_call(*, cap: int = 5, content: str = "x") -> StageCall: +def _stage_call( + *, + cap: int = 5, + content: str = "x", + output_contract: OutputContract | None = None, +) -> StageCall: return StageCall( stage="initial", provider_id="fictional-provider", @@ -48,6 +54,7 @@ def _stage_call(*, cap: int = 5, content: str = "x") -> StageCall: model_revision="fixture-r1", messages=(ChatMessage(role="user", content=content),), max_output_tokens=cap, + output_contract=output_contract, ) @@ -89,6 +96,38 @@ async def fake_call_model(name, model_id, messages, **kwargs): assert len(checkpoints[-1][1]) == 1 +@pytest.mark.asyncio +async def test_gateway_passes_stage_output_contract_to_provider() -> None: + contract = OutputContract( + schema={"type": "object", "properties": {}, "additionalProperties": False}, + schema_name="FixtureContract", + strict=True, + ) + observed: list[OutputContract | None] = [] + + async def fake_call_model(name, model_id, messages, **kwargs): + del messages + observed.append(kwargs.get("output_contract")) + return ModelAnswer( + name=name, + model_id=model_id, + answer="{}", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + client = LiveProviderClient( + price_book=_price_book(input_rate="1", output_rate="1"), + hard_cap_usd=Decimal("1"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + call = _stage_call(output_contract=contract) + + await client.call(call) + + assert observed == [contract] + + @pytest.mark.asyncio async def test_gateway_allows_only_one_in_flight_call() -> None: active = 0 diff --git a/tests/evals/test_live_protocols.py b/tests/evals/test_live_protocols.py index a27caef..1b70bc9 100644 --- a/tests/evals/test_live_protocols.py +++ b/tests/evals/test_live_protocols.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json from collections.abc import Sequence import pytest @@ -16,6 +17,7 @@ from conclave.evals.models import EVAL_CONDITION_IDS, ProviderModelSpec, PublicTask from conclave.evals.protocols import live_protocol_registry from conclave.models import ELITE_MIN_RESPONDERS, ModelAnswer +from conclave.verdict import CouncilVerdict, verdict_extraction_json_schema def _roster(size: int = 3) -> tuple[ProviderModelSpec, ...]: @@ -56,6 +58,12 @@ async def call(self, call: StageCall) -> ModelAnswer: model_id=call.model_id, error="bounded fake failure", ) + if call.stage in {"verdict", "verdict_repair"}: + return ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + answer=_valid_verdict_extraction(), + ) return ModelAnswer( name=call.provider_id, model_id=call.model_id, @@ -63,6 +71,64 @@ async def call(self, call: StageCall) -> ModelAnswer: ) +def _valid_verdict_extraction() -> str: + return json.dumps( + { + "verdict_applies": True, + "verdict_type": "decision", + "headline": "Choose option A.", + "recommendation": "Adopt option A with the stated safeguards.", + "positions": [ + { + "label": "option-a", + "summary": "Prefer option A.", + "providers": ["Model A", "Model B"], + "evidence_answer_ids": ["answer-a", "answer-b"], + }, + { + "label": "option-b", + "summary": "Prefer option B.", + "providers": ["Model C"], + "evidence_answer_ids": ["answer-c"], + }, + ], + "provider_votes": [ + {"provider": "Model A", "position_label": "option-a"}, + {"provider": "Model B", "position_label": "option-a"}, + {"provider": "Model C", "position_label": "option-b"}, + ], + "conflicts": [ + { + "topic": "preferred option", + "position_labels": ["option-a", "option-b"], + "summary": "The panel split between A and B.", + } + ], + "minority_reports": [], + "caveats": [], + # The canonical engine must ignore this model-emitted number and + # deterministically recompute 2 / 3 from the provider votes. + "consensus_score": 0.01, + } + ) + + +class ScriptedVerdictClient(RecordingSequentialClient): + def __init__(self, *verdict_responses: str) -> None: + super().__init__() + self._verdict_responses = iter(verdict_responses) + + async def call(self, call: StageCall) -> ModelAnswer: + if call.stage not in {"verdict", "verdict_repair"}: + return await super().call(call) + self.calls.append(call) + return ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + answer=next(self._verdict_responses), + ) + + def _message_text(call: StageCall) -> str: return "\n".join(message.content for message in call.messages) @@ -71,15 +137,20 @@ def test_live_registry_covers_exactly_six_versioned_conditions() -> None: registry = live_protocol_registry() assert tuple(registry) == EVAL_CONDITION_IDS - assert all(entry.protocol_version.endswith("_v1") for entry in registry.values()) + assert all(entry.protocol_version.endswith("_v2") for entry in registry.values()) assert set(LIVE_PROMPT_VERSIONS) == set(EVAL_CONDITION_IDS) assert all(LIVE_PROMPT_VERSIONS[condition] for condition in EVAL_CONDITION_IDS) def test_stage_caps_are_positive_deterministic_and_sum_to_cell_ceiling() -> None: + import conclave.evals.live_protocols as live_protocols + + assert live_protocols.LIVE_STAGE_MINIMUM_CAPS_VERSION == "live_stage_minimum_caps_v1" + assert all(cap > 1 for cap in live_protocols.LIVE_STAGE_MINIMUM_CAPS.values()) for condition_id in EVAL_CONDITION_IDS: call_count = len(stage_call_sequence(condition_id, roster_size=3)) - ceiling = call_count * 17 + 5 + minimum = live_protocols.minimum_cell_ceiling(condition_id, roster_size=3) + ceiling = minimum + call_count * 17 + 5 first = allocate_stage_caps(condition_id, roster_size=3, cell_ceiling=ceiling) second = allocate_stage_caps(condition_id, roster_size=3, cell_ceiling=ceiling) @@ -88,8 +159,73 @@ def test_stage_caps_are_positive_deterministic_and_sum_to_cell_ceiling() -> None assert len(first) == call_count assert all(cap > 0 for cap in first) assert sum(first) == ceiling - equal_share, remainder = divmod(ceiling, call_count) - assert first[-1] == equal_share + remainder + assert all( + cap >= live_protocols.LIVE_STAGE_MINIMUM_CAPS[stage] + for cap, (stage, _member_index) in zip( + first, + stage_call_sequence(condition_id, roster_size=3), + strict=True, + ) + ) + + +@pytest.mark.asyncio +async def test_elite_uses_guarded_canonical_verdict_and_skips_repair_when_valid() -> None: + client = ScriptedVerdictClient(_valid_verdict_extraction()) + + output = await execute_live_condition( + "elite_full", + task=_task(), + roster=_roster(), + cell_ceiling=6144, + client=client, + ) + + verdict_calls = [call for call in client.calls if call.stage.startswith("verdict")] + assert [call.stage for call in verdict_calls] == ["verdict"] + assert verdict_calls[0].output_contract is not None + assert verdict_calls[0].output_contract.strict is True + assert verdict_calls[0].output_contract.schema == verdict_extraction_json_schema() + verdict = CouncilVerdict.model_validate_json(output) + assert verdict.recommendation == "Adopt option A with the stated safeguards." + assert verdict.consensus_score == pytest.approx(2 / 3) + assert verdict.consensus_label == "majority" + + +@pytest.mark.asyncio +async def test_elite_repairs_malformed_verdict_once_through_guarded_gateway() -> None: + client = ScriptedVerdictClient("not-json", _valid_verdict_extraction()) + + output = await execute_live_condition( + "elite_full", + task=_task(), + roster=_roster(), + cell_ceiling=6144, + client=client, + ) + + verdict_calls = [call for call in client.calls if call.stage.startswith("verdict")] + assert [call.stage for call in verdict_calls] == ["verdict", "verdict_repair"] + assert all(call.output_contract is not None for call in verdict_calls) + assert "could not be used" in _message_text(verdict_calls[-1]) + assert CouncilVerdict.model_validate_json(output).consensus_score == pytest.approx(2 / 3) + + +@pytest.mark.asyncio +async def test_elite_failed_verdict_repair_is_non_success_and_never_returns_raw_text() -> None: + client = ScriptedVerdictClient("first malformed", "repair also malformed") + + with pytest.raises(ValueError, match="canonical CouncilVerdict"): + await execute_live_condition( + "elite_full", + task=_task(), + roster=_roster(), + cell_ceiling=6144, + client=client, + ) + + verdict_calls = [call for call in client.calls if call.stage.startswith("verdict")] + assert [call.stage for call in verdict_calls] == ["verdict", "verdict_repair"] @pytest.mark.asyncio @@ -106,7 +242,7 @@ async def test_single_and_self_refine_use_only_frozen_lead_member() -> None: condition_id, task=_task(), roster=roster, - cell_ceiling=120, + cell_ceiling=1024, client=client, ) @@ -151,12 +287,17 @@ async def test_multi_model_conditions_call_members_in_frozen_order() -> None: condition_id, task=_task(), roster=roster, - cell_ceiling=1100, + cell_ceiling=6144, client=client, ) assert [(call.stage, call.provider_id) for call in client.calls] == expected_calls - assert output == f"decision-artifact:{expected_calls[-1][0]}:{len(expected_calls)}" + if condition_id == "elite_full": + assert CouncilVerdict.model_validate_json(output).consensus_score == pytest.approx( + 2 / 3 + ) + else: + assert output == f"decision-artifact:{expected_calls[-1][0]}:{len(expected_calls)}" assert client.max_active == 1 @@ -170,7 +311,7 @@ async def test_critique_revision_and_elite_prompts_are_anonymized() -> None: condition_id, task=_task(), roster=roster, - cell_ceiling=1100, + cell_ceiling=6144, client=client, ) @@ -209,7 +350,7 @@ def tracked_revision(*args, **kwargs): "elite_full", task=_task(), roster=roster, - cell_ceiling=1100, + cell_ceiling=6144, client=client, ) @@ -223,7 +364,7 @@ def tracked_revision(*args, **kwargs): "elite_full", task=_task(), roster=roster, - cell_ceiling=1100, + cell_ceiling=6144, client=failed, ) assert [call.stage for call in failed.calls] == ["initial"] * 3 @@ -234,7 +375,7 @@ def tracked_revision(*args, **kwargs): "elite_full", task=_task(), roster=_roster(2), - cell_ceiling=1100, + cell_ceiling=6144, client=too_small_roster_client, ) assert too_small_roster_client.calls == [] @@ -242,11 +383,13 @@ def tracked_revision(*args, **kwargs): @pytest.mark.asyncio async def test_too_small_cell_budget_fails_before_any_call() -> None: + import conclave.evals.live_protocols as live_protocols + roster = _roster() for condition_id in EVAL_CONDITION_IDS: client = RecordingSequentialClient() - too_small = len(stage_call_sequence(condition_id, roster_size=len(roster))) - 1 + too_small = live_protocols.minimum_cell_ceiling(condition_id, roster_size=len(roster)) - 1 with pytest.raises(ValueError, match="too small"): await execute_live_condition( diff --git a/tests/evals/test_live_replay.py b/tests/evals/test_live_replay.py index 2ac494f..0a70c12 100644 --- a/tests/evals/test_live_replay.py +++ b/tests/evals/test_live_replay.py @@ -75,8 +75,43 @@ async def delegate(url, headers, body, timeout): assert headers["Authorization"] == f"Bearer {FAKE_KEY}" calls.append((url, body["model"], body["max_tokens"])) ordinal = len(calls) + content = f"Fictional decision artifact {ordinal:03d}." + if any( + "verdict extractor" in message.get("content", "") + for message in body.get("messages", []) + ): + content = json.dumps( + { + "verdict_applies": True, + "verdict_type": "decision", + "headline": "Choose the fictional safe route.", + "recommendation": "Use Route A with the stated safeguards.", + "positions": [ + { + "label": "route-a", + "summary": "Route A is preferred.", + "providers": ["Model A", "Model B", "Model C"], + "evidence_answer_ids": [ + "fixture-a", + "fixture-b", + "fixture-c", + ], + } + ], + "provider_votes": [ + {"provider": "Model A", "position_label": "route-a"}, + {"provider": "Model B", "position_label": "route-a"}, + {"provider": "Model C", "position_label": "route-a"}, + ], + "conflicts": [], + "minority_reports": [], + "caveats": [], + }, + separators=(",", ":"), + sort_keys=True, + ) return 200, { - "choices": [{"message": {"content": f"Fictional decision artifact {ordinal:03d}."}}], + "choices": [{"message": {"content": content}}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, } @@ -192,6 +227,7 @@ async def test_live_smoke_replay_executes_all_conditions_with_zero_network_calls planned.condition_id, roster_size=len(roster.members), ) + if stage != "verdict_repair" ) actual_call_order = [ (receipt.stage, receipt.provider_id, receipt.model_id) diff --git a/tests/evals/test_live_runner.py b/tests/evals/test_live_runner.py index 20888be..9fd25ef 100644 --- a/tests/evals/test_live_runner.py +++ b/tests/evals/test_live_runner.py @@ -37,6 +37,34 @@ DIGEST = "sha256:" + "a" * 64 HARD_CAP = Decimal("10.00") +LIVE_FIXTURE_CELL_CEILING = 6144 + + +def _valid_verdict_extraction_text() -> str: + return json.dumps( + { + "verdict_applies": True, + "verdict_type": "decision", + "headline": "Choose the fictional safe option.", + "recommendation": "Use the fictional safe option with safeguards.", + "positions": [ + { + "label": "safe-option", + "summary": "The fictional safe option is preferred.", + "providers": ["Model A", "Model B", "Model C"], + "evidence_answer_ids": ["fixture-a", "fixture-b", "fixture-c"], + } + ], + "provider_votes": [ + {"provider": "Model A", "position_label": "safe-option"}, + {"provider": "Model B", "position_label": "safe-option"}, + {"provider": "Model C", "position_label": "safe-option"}, + ], + "conflicts": [], + "minority_reports": [], + "caveats": [], + } + ) def _rosters(*, members_per_roster: int = 3) -> tuple[RosterSpec, ...]: @@ -64,6 +92,7 @@ def _live_inputs( evidence_classification: str = "paid_exploratory_pilot", approved_spend_ceiling_usd: float = 10.0, version_drift: bool = False, + cell_ceiling: int = LIVE_FIXTURE_CELL_CEILING, ): tasks = [ PublicTask( @@ -136,18 +165,22 @@ def _live_inputs( tasks=tasks, replicates=1, seed=20260718, - output_token_budgets=dict.fromkeys(CONDITION_IDS, 64), + output_token_budgets=dict.fromkeys(CONDITION_IDS, cell_ceiling), frozen_design=design, ) return tasks, manifest, price_book async def _successful_provider(name, model_id, messages, **kwargs): - del messages, kwargs + del messages return ModelAnswer( name=name, model_id=model_id, - answer=f"fictional decision from {name}", + answer=( + _valid_verdict_extraction_text() + if kwargs.get("output_contract") is not None + else f"fictional decision from {name}" + ), usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), ) @@ -225,6 +258,7 @@ async def forbidden_provider(*args, **kwargs): invalid_cases.append(_live_inputs(approved_spend_ceiling_usd=9.99)) invalid_cases.append(_live_inputs(members_per_roster=2)) invalid_cases.append(_live_inputs(version_drift=True)) + invalid_cases.append(_live_inputs(cell_ceiling=1024)) for index, (tasks, manifest, price_book) in enumerate(invalid_cases): with pytest.raises(RunValidationError): @@ -307,6 +341,7 @@ async def recording_provider(name, model_id, messages, **kwargs): assert first_seen_ids == expected_ids assert call_counts == { run.planned_run_id: len(stage_call_sequence(run.condition_id, roster_size=3)) + - (1 if run.condition_id == "elite_full" else 0) for run in manifest.planned_runs } assert tuple(record.planned_run_id for record in study_run.records) == expected_ids @@ -436,7 +471,15 @@ async def provider(name, model_id, messages, **kwargs): if failure_mode == "timeout": raise TimeoutError("fictional provider timed out") if failure_mode == "missing_usage": - return ModelAnswer(name=name, model_id=model_id, answer="unmetered fixture answer") + return ModelAnswer( + name=name, + model_id=model_id, + answer=( + _valid_verdict_extraction_text() + if kwargs.get("output_contract") is not None + else "unmetered fixture answer" + ), + ) if failure_mode == "provider_error": return ModelAnswer(name=name, model_id=model_id, error="fictional provider error") return ModelAnswer( diff --git a/tests/fixtures/evals/live_smoke/manifest.json b/tests/fixtures/evals/live_smoke/manifest.json index 9161f00..1c77342 100644 --- a/tests/fixtures/evals/live_smoke/manifest.json +++ b/tests/fixtures/evals/live_smoke/manifest.json @@ -75,12 +75,12 @@ "elite_full": "elite_1_synthesis_2026-06-29_verdict_2" }, "condition_protocol_versions": { - "single_frontier": "single_frontier_live_v1", - "self_refine": "self_refine_live_v1", - "independent_synthesis": "independent_synthesis_live_v1", - "critique_only": "critique_only_live_v1", - "revision_only": "revision_only_live_v1", - "elite_full": "elite_full_live_v1" + "single_frontier": "single_frontier_live_v2", + "self_refine": "self_refine_live_v2", + "independent_synthesis": "independent_synthesis_live_v2", + "critique_only": "critique_only_live_v2", + "revision_only": "revision_only_live_v2", + "elite_full": "elite_full_live_v2" }, "generation_settings_hash": "sha256:3333333333333333333333333333333333333333333333333333333333333333", "evaluator_version": "fictional-live-evaluator-v1", @@ -146,127 +146,127 @@ "preregistration_id": null, "preregistration_hash": null }, - "frozen_design_hash": "sha256:7b8fc1a68dd70af8538d61af258d84dfb16f97b744b8fa584b45a12ea352e9d0", + "frozen_design_hash": "sha256:1870bb73e7e57f7b762cbd76898ab07267c1948a799f01b86a5e05a9be9581af", "planned_runs": [ { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_c89136045f317beb2776b8a7", + "planned_run_id": "run_1514260bbb609559e7fac798", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", "condition_id": "revision_only", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_26d60cf8204277ed433cf693", + "planned_run_id": "run_250a6a43f71c819882567f2f", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", "condition_id": "elite_full", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_2d94e5980d36edfdd6f924ef", + "planned_run_id": "run_03e305fc346582f50bd98a86", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", "condition_id": "independent_synthesis", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_6354ef6a4d3a3d6af3b0ba5b", + "planned_run_id": "run_5ab2ccbb3c51c7c9b798847b", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", "condition_id": "self_refine", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_0677ca748602fc84dd97a6f2", + "planned_run_id": "run_56ace8153142d9801dc5d6cb", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", "condition_id": "critique_only", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_fbf0a2094bd7071b3fd15766", + "planned_run_id": "run_9663dc1ddd27fe7f45e59c47", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", "condition_id": "single_frontier", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_12f47bc053d373a70c36e190", + "planned_run_id": "run_c75cdbf92f852ab704e4e831", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", "condition_id": "elite_full", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_22980c01680221e2a1af883f", + "planned_run_id": "run_0dad25939ce03c3ca19f7dcf", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", "condition_id": "self_refine", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_2d41f152248bbbe7b0a872f3", + "planned_run_id": "run_a6934c752d97ad8fced5c4b4", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", "condition_id": "revision_only", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_5aa1617957acbe77c6e416a1", + "planned_run_id": "run_1e19e08e1ce475e8d6fd9e64", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", "condition_id": "critique_only", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_c0acdec7f0d30a4b7db4e7b5", + "planned_run_id": "run_fa0d8ba45a49c759dd057670", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", "condition_id": "independent_synthesis", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_9354497a641db13eb5d91cdd", + "planned_run_id": "run_f789cd855c22413292bf37be", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", "condition_id": "single_frontier", "replicate": 1, - "max_output_tokens": 64 + "max_output_tokens": 6144 } ] } diff --git a/tests/fixtures/evals/live_smoke/replay.json b/tests/fixtures/evals/live_smoke/replay.json index c38778d..434c58e 100644 --- a/tests/fixtures/evals/live_smoke/replay.json +++ b/tests/fixtures/evals/live_smoke/replay.json @@ -1,14 +1,14 @@ { "schema_version": "conclave_replay_v1", - "base_manifest_hash": "sha256:456096f88d22398553a1d0cc6024c8ed469f9115a28731f30ce4d1d960155442", + "base_manifest_hash": "sha256:72472cd0faa0c9de49a195ae70a797a4c763e6e20c96ad982a89bb4bfc55a076", "records": [ { - "request_hash": "sha256:8e116cb7c77c18cbe04219e375cd95923cb30625aec753fbc4a4a3ca2907dea3", + "request_hash": "sha256:cc57d8d8ed8963f3537727956243f353427936b20d5365ebb8438e91a93340e1", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 786, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -36,12 +36,12 @@ } }, { - "request_hash": "sha256:f97d7447f59a3a1923b83ecdbd2a256e788aa0306b80e34487c82f33195cc1c8", + "request_hash": "sha256:2e653c544f6ead81a481fad8280b49024ade5526a96780e6af5a76f9cf41c3d3", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 786, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -69,12 +69,12 @@ } }, { - "request_hash": "sha256:9a6ce2607e0eb30c7477f008399efde19c5aeed1b7fa1c061620db31b2946fa9", + "request_hash": "sha256:7fe02d317cbc5fd6bc862df80f6418ba7d1cdbd0aac145add051fcd20c6ad575", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 786, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -102,12 +102,12 @@ } }, { - "request_hash": "sha256:971312b5299013bf65b12468532d0cf1a21dcbfd6f5813b465b5d1e4ed3780f8", + "request_hash": "sha256:7a55e1ca1bc4fd291414e0cf89e360235bb902b1b16d8d38913ad7915c2b39f9", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 914, "messages": [ { "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 001.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 001.\n\n### Model B peer answer\nFictional decision artifact 002.\n\n### Model C peer answer\nFictional decision artifact 003.\n\nProduce one complete revised answer to the original prompt.", @@ -135,12 +135,12 @@ } }, { - "request_hash": "sha256:9e438ed4b52535986e2e1e9353d84315ed6390a1af4cba5d49f56ac13c8c8bf8", + "request_hash": "sha256:588214dba5cab890058699ad3aaa3154651feb222b108c1515144f443ad3002a", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 914, "messages": [ { "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 002.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 001.\n\n### Model B peer answer\nFictional decision artifact 002.\n\n### Model C peer answer\nFictional decision artifact 003.\n\nProduce one complete revised answer to the original prompt.", @@ -168,12 +168,12 @@ } }, { - "request_hash": "sha256:7da3f22b52b7040fe3329c7b54d8c9ee33c64d5f129d9f2457d32422d69bb284", + "request_hash": "sha256:f06b82738ea98cca45d7f979b4091aa8bc5133875063d7bf3b18d8aa0aad35b3", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 914, "messages": [ { "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 003.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 001.\n\n### Model B peer answer\nFictional decision artifact 002.\n\n### Model C peer answer\nFictional decision artifact 003.\n\nProduce one complete revised answer to the original prompt.", @@ -201,12 +201,12 @@ } }, { - "request_hash": "sha256:006fcd47eb736ca44e176c41ec3f123db8fc74f74474ff0d859c7ff21e4bf493", + "request_hash": "sha256:d508a5497c969942dcf7bffa205bd1a77000cdb442d535b76516e185b5db6436", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 10, + "max_tokens": 1044, "messages": [ { "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", @@ -238,12 +238,12 @@ } }, { - "request_hash": "sha256:c06913d309bff2c3b15ee8978b1776b0f9bfa776d681d8491e394f0fffa0d41c", + "request_hash": "sha256:e1ba0136778f7688e5aa74728c7e05aeca18860110e12717b4b5d62892c9ee8a", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -271,12 +271,12 @@ } }, { - "request_hash": "sha256:84517758489cf8ff0b17c73e5067d71fcc01244d338af04f4ad5bdca2c016df3", + "request_hash": "sha256:7a4b57d6c3ba105284afee582de680337d30a318593f683859f5ca75fb4af59b", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -304,12 +304,12 @@ } }, { - "request_hash": "sha256:3fdbe8d950bdeec566d42580d4f3f09f99a160cdd3ce6b1c4c07386489d2d8cc", + "request_hash": "sha256:f505a9749b9369d62d22e46166b8f5fe6ddbeb6ca51e7bb33768136ef4a4fbab", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -337,12 +337,12 @@ } }, { - "request_hash": "sha256:50912c3ad490b25f600bfe77424145b215768f2ef862cadf0caffca899e812d2", + "request_hash": "sha256:e50f2b0ea8373f8beac8118bd2c820dafbc569f622fa9367bd219cd20d04529c", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -374,12 +374,12 @@ } }, { - "request_hash": "sha256:89a1c6f02c94f203c449bae2c49321df5738cf3e999cc3f1ab2394990655fabf", + "request_hash": "sha256:77cb17fe1404da194cb09e6db9ae44629f1c1dd1d318ddafa619ffc15ab4c0aa", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -411,12 +411,12 @@ } }, { - "request_hash": "sha256:f31454bbd91ab0005217e1c9a12369ea7009cf6d79c9901c904897816c2616c7", + "request_hash": "sha256:abdced28c27045858bac32c56adac3ff7e5373a44beb2f3a5c38e358a2f36c3e", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -448,12 +448,12 @@ } }, { - "request_hash": "sha256:1d7cc1836d4d58b9be1f8fc3db663f862f2aa72012e2a3cebed3d5a801c46224", + "request_hash": "sha256:2754e419e2382fd7535aa1f4d50d3bd73adc659a1b208b565849e2000d3823e8", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 501, "messages": [ { "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", @@ -485,12 +485,12 @@ } }, { - "request_hash": "sha256:316eb69ec8c0cc581685f457d5b52403220660c5246bff6bd1c0e190023544c6", + "request_hash": "sha256:0db5d3ad3682b36e7e52f3a4d3cf7ec05d46aa9647881d19f11a8b82b11a30cc", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 501, "messages": [ { "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", @@ -522,12 +522,12 @@ } }, { - "request_hash": "sha256:f1e1a14b5c6911a5da18d3ce1506e26e9d20383780714fc5346da584c9d96884", + "request_hash": "sha256:d956686457a298cba78ed4fa28827f50e73a5f69f4ebbdb346406f5edffbab19", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 501, "messages": [ { "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", @@ -559,12 +559,12 @@ } }, { - "request_hash": "sha256:fe72eae35cdb78ab15a438a081f37c978b8852489cd369510aebc26af5f4ed35", + "request_hash": "sha256:322e700e4cc5f29ca0bf46953fed2294effc18c8f918d649e0a81d30a5089e6c", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 629, "messages": [ { "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", @@ -596,19 +596,19 @@ } }, { - "request_hash": "sha256:85c6a8f75e6d3685ec036be0e01fc80aa0835cf817f57286110971c4987370cd", + "request_hash": "sha256:6f76bf698fc495f0e86e9bf221ac30de80c42dfcdece486f3e2ad2f506a0a5e8", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 14, + "max_tokens": 889, "messages": [ { - "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Add conflicts[] only when there are two or more positions in tension.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number \u2014 the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", + "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Add conflicts[] only when there are two or more positions in tension.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number โ€” the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", "role": "system" }, { - "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_30f27d6d1f3068239a264a92) \u2014 from Model A\nFictional decision artifact 014.\n\n### Member answer (answer id: ca_revision_54bce3f164f68c9ddb3effdf) \u2014 from Model B\nFictional decision artifact 015.\n\n### Member answer (answer id: ca_revision_648c3c5d70c342eaa5812f6e) \u2014 from Model C\nFictional decision artifact 016.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"consensus_score\": {\n \"type\": [\n \"number\",\n \"null\"\n ]\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n },\n \"confidence\": {\n \"type\": \"string\",\n \"enum\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"verdict_applies\"\n ]\n}", + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_30f27d6d1f3068239a264a92) โ€” from Model A\nFictional decision artifact 014.\n\n### Member answer (answer id: ca_revision_54bce3f164f68c9ddb3effdf) โ€” from Model B\nFictional decision artifact 015.\n\n### Member answer (answer id: ca_revision_648c3c5d70c342eaa5812f6e) โ€” from Model C\nFictional decision artifact 016.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"consensus_score\": {\n \"type\": [\n \"number\",\n \"null\"\n ]\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n },\n \"confidence\": {\n \"type\": \"string\",\n \"enum\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"verdict_applies\"\n ]\n}", "role": "user" } ], @@ -621,7 +621,7 @@ "choices": [ { "message": { - "content": "Fictional decision artifact 018." + "content": "{\"caveats\":[],\"conflicts\":[],\"headline\":\"Choose the fictional safe route.\",\"minority_reports\":[],\"positions\":[{\"evidence_answer_ids\":[\"fixture-a\",\"fixture-b\",\"fixture-c\"],\"label\":\"route-a\",\"providers\":[\"Model A\",\"Model B\",\"Model C\"],\"summary\":\"Route A is preferred.\"}],\"provider_votes\":[{\"position_label\":\"route-a\",\"provider\":\"Model A\"},{\"position_label\":\"route-a\",\"provider\":\"Model B\"},{\"position_label\":\"route-a\",\"provider\":\"Model C\"}],\"recommendation\":\"Use Route A with the stated safeguards.\",\"verdict_applies\":true,\"verdict_type\":\"decision\"}" } } ], @@ -633,12 +633,12 @@ } }, { - "request_hash": "sha256:66b06f34e69ffb9fa4bfaa79969147bd90b1c424dfd19a950c80c57a953eeb0b", + "request_hash": "sha256:2f49af35b10a45083cec38a6929c4f24840fd9ddb9a0e8337e3e1840ad0e120c", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 16, + "max_tokens": 1472, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -666,12 +666,12 @@ } }, { - "request_hash": "sha256:cfabe1fcf9f9aa97f5ce3c6f5fd00c0b58e0d19b6328b0858efe8559d145ca8a", + "request_hash": "sha256:3e871bdcc4a67bae1bba8dc450a253797ad6d689350c9cc0ae1bfec5b8e79f58", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 16, + "max_tokens": 1472, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -699,12 +699,12 @@ } }, { - "request_hash": "sha256:64c25574dc75e5cb8785b3accba9028868bff1e40e1f871b2c7dff341d1958a1", + "request_hash": "sha256:1272f0bc2baa43e27e852787219b6b445ff23481fe9d7785e88c63a72dea8a2a", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 16, + "max_tokens": 1472, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -732,12 +732,12 @@ } }, { - "request_hash": "sha256:c4806c924f37455be442b090adf8b2137c31e00568c74b22d006a27465541af4", + "request_hash": "sha256:d2145371bbde2a6c3a79ce2b89ec7b4aa8e69a738a9566bd0d51766a89799671", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 16, + "max_tokens": 1728, "messages": [ { "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", @@ -769,12 +769,12 @@ } }, { - "request_hash": "sha256:ad331e55c84a2f6c683a6bae288410db79d8e2a96bb5eaafec8e77a16b6be353", + "request_hash": "sha256:b9e89bcfe9000abe09e1a09cc023d3168adf7e6c5dfa45e34eb224ca3862cc7a", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 32, + "max_tokens": 3008, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -802,12 +802,12 @@ } }, { - "request_hash": "sha256:af1a0c246e565e9909111e9dc104f3e2c90de91a9009607c3b32e848383e60c6", + "request_hash": "sha256:a60349a2d5cfdf86fc62e921951f4eca9182a8c026469e618d05b63ffc11b947", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 32, + "max_tokens": 3136, "messages": [ { "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour draft:\nFictional decision artifact 023.\n\nProduce one complete, improved final answer.", @@ -835,12 +835,12 @@ } }, { - "request_hash": "sha256:8e116cb7c77c18cbe04219e375cd95923cb30625aec753fbc4a4a3ca2907dea3", - "occurrence_index": 1, + "request_hash": "sha256:d948e25d735ca60fa12fa1f58a1a13a3b83756c89a14cb6aec3f0636dd8c1d79", + "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -868,12 +868,12 @@ } }, { - "request_hash": "sha256:f97d7447f59a3a1923b83ecdbd2a256e788aa0306b80e34487c82f33195cc1c8", - "occurrence_index": 1, + "request_hash": "sha256:85c1e9279d0f7807e5b2e11ee8e5b7598b6c0cd3e6902a62af28977b29770f8f", + "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -901,12 +901,12 @@ } }, { - "request_hash": "sha256:9a6ce2607e0eb30c7477f008399efde19c5aeed1b7fa1c061620db31b2946fa9", - "occurrence_index": 1, + "request_hash": "sha256:94eb52642fa244a450a865a90d5f84760985a99296505bb7c65d927faeb43f27", + "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -934,12 +934,12 @@ } }, { - "request_hash": "sha256:a1f4e1656f2834e946475854cb1f9ac9088f44faca5f545fd3cf2da08d56432a", + "request_hash": "sha256:0e527c0459e3bd7f839e6308337fce1e7b8b1d869901ca1f8b53861c5a282c3f", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -971,12 +971,12 @@ } }, { - "request_hash": "sha256:4a884b70e16a4838b8a7113c53fa069413cf91549de543d3f6acadb2eacf8510", + "request_hash": "sha256:5ecd7802617269a8b949485af7d35935b8c528a4961623c7ebabc273e8154060", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -1008,12 +1008,12 @@ } }, { - "request_hash": "sha256:46dcb70a5b05062e46f157a91a70efe730961577dbddbbb5382440e5eaa3f17e", + "request_hash": "sha256:929b7a97cc9664a70b2e04343b559da4243e62c08b9993455f6bd4e425dfb1bd", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -1045,12 +1045,12 @@ } }, { - "request_hash": "sha256:a9b39c9034341d9a625f1a1369f341650c55f188a3da89ccaf3ebb62b17eb6f8", + "request_hash": "sha256:46a847386e76edb863614a44b87ceb3e684fdd2c87b617cd1aeecb2bdc324e68", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 10, + "max_tokens": 1098, "messages": [ { "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", @@ -1082,12 +1082,12 @@ } }, { - "request_hash": "sha256:6fad3d158dc6fbf6b5d4ec4ee9206bc152d6e18d4aecde8b6c7b5997db5bf47f", + "request_hash": "sha256:430c93c9a1dffaddae7ee9001fc1e2246c3b8cba21818d2e0076d1f5f8ca4f12", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 64, + "max_tokens": 6144, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1115,12 +1115,12 @@ } }, { - "request_hash": "sha256:3fdbe8d950bdeec566d42580d4f3f09f99a160cdd3ce6b1c4c07386489d2d8cc", + "request_hash": "sha256:f505a9749b9369d62d22e46166b8f5fe6ddbeb6ca51e7bb33768136ef4a4fbab", "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1148,12 +1148,12 @@ } }, { - "request_hash": "sha256:84517758489cf8ff0b17c73e5067d71fcc01244d338af04f4ad5bdca2c016df3", + "request_hash": "sha256:7a4b57d6c3ba105284afee582de680337d30a318593f683859f5ca75fb4af59b", "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1181,12 +1181,12 @@ } }, { - "request_hash": "sha256:c06913d309bff2c3b15ee8978b1776b0f9bfa776d681d8491e394f0fffa0d41c", + "request_hash": "sha256:e1ba0136778f7688e5aa74728c7e05aeca18860110e12717b4b5d62892c9ee8a", "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1214,12 +1214,12 @@ } }, { - "request_hash": "sha256:52b77096e212f4eecdbdcb75ff2bca28fb0f8b2c686ffff6fd11dea29da0807d", + "request_hash": "sha256:8068ebf0f001d22e6d380dea58d25cc6dc3915e38da091d270e7796bbc743163", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -1251,12 +1251,12 @@ } }, { - "request_hash": "sha256:a835c8acb9a3f1406ba7d7d7ab59cb1db5c4b1c2aae78ecb1388742c5bcef071", + "request_hash": "sha256:c779f8767450873543244a7a5c9f54dda6e2d6c8fb2768c73c07fb75db95e188", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -1288,12 +1288,12 @@ } }, { - "request_hash": "sha256:8fe9864ae4f5edff442dcc1056248af7878d4aa676744d94f08d252b49c2b080", + "request_hash": "sha256:a9d86a7c438ee7bb6cc42fc76482c67d418fc28dec2d1c380166b81a237040a8", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 373, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -1325,12 +1325,12 @@ } }, { - "request_hash": "sha256:9259e556a6b7ccff0fb00fa19df61c2ceac242d99ac3823083c166e32ca4e2fb", + "request_hash": "sha256:9eb9cf379805f0a56de74c478065cb163150a2e615a1e8bcc9473ddeee00445d", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 501, "messages": [ { "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", @@ -1362,12 +1362,12 @@ } }, { - "request_hash": "sha256:94c516d6931c2ed49b1bb6c232b976ba9485c6e2e3affcab7a6f5165bdd01a39", + "request_hash": "sha256:75e3638df32734081c0d982188db9f850bd498440810d2926b346c61d58203b9", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 501, "messages": [ { "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", @@ -1399,12 +1399,12 @@ } }, { - "request_hash": "sha256:be1accab0b033bc49fc04b41745d9ba84106bf32c6a9e84bd84dde31ff205427", + "request_hash": "sha256:ec6a389e0cec2bbb6da227d889821d67c901a96075f2ac33f7b6bdb24ca8fb9d", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 501, "messages": [ { "content": "You are revising your independent answer after a claim audit by an elite decision council. Use the anonymized initial panel and critiques to correct unsupported claims, resolve conflicts where the supplied material permits, and clearly mark externally unverified claims. Do not invent citations, sources, quotations, measurements, or facts. Produce a complete standalone answer to the original prompt, not a description of your edits.", @@ -1436,12 +1436,12 @@ } }, { - "request_hash": "sha256:02bda1be5907a6e502ad841eafee3ad205b35015ffd1749557f414f6348f74ab", + "request_hash": "sha256:eca9f8611504368ff8d080c980410564d9c606f61bcf538983c80e7153308324", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 5, + "max_tokens": 629, "messages": [ { "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", @@ -1473,19 +1473,19 @@ } }, { - "request_hash": "sha256:5df9018dc5065ede8d143ca207798c7b9213ca24c76e9a0446723912ec0618ff", + "request_hash": "sha256:986a343856a9e004b40161d47a0a8adf416114e9aa84cb6f8acada2ea9371286", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 14, + "max_tokens": 889, "messages": [ { - "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Add conflicts[] only when there are two or more positions in tension.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number \u2014 the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", + "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Add conflicts[] only when there are two or more positions in tension.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number โ€” the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", "role": "system" }, { - "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_52b860f7867fc83421bd3ee9) \u2014 from Model A\nFictional decision artifact 039.\n\n### Member answer (answer id: ca_revision_7b938c9dc355e2347e03ced7) \u2014 from Model B\nFictional decision artifact 040.\n\n### Member answer (answer id: ca_revision_9b8324a46592779e0bad8aa1) \u2014 from Model C\nFictional decision artifact 041.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"consensus_score\": {\n \"type\": [\n \"number\",\n \"null\"\n ]\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n },\n \"confidence\": {\n \"type\": \"string\",\n \"enum\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"verdict_applies\"\n ]\n}", + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_52b860f7867fc83421bd3ee9) โ€” from Model A\nFictional decision artifact 039.\n\n### Member answer (answer id: ca_revision_7b938c9dc355e2347e03ced7) โ€” from Model B\nFictional decision artifact 040.\n\n### Member answer (answer id: ca_revision_9b8324a46592779e0bad8aa1) โ€” from Model C\nFictional decision artifact 041.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"consensus_score\": {\n \"type\": [\n \"number\",\n \"null\"\n ]\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n },\n \"confidence\": {\n \"type\": \"string\",\n \"enum\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"verdict_applies\"\n ]\n}", "role": "user" } ], @@ -1498,7 +1498,7 @@ "choices": [ { "message": { - "content": "Fictional decision artifact 043." + "content": "{\"caveats\":[],\"conflicts\":[],\"headline\":\"Choose the fictional safe route.\",\"minority_reports\":[],\"positions\":[{\"evidence_answer_ids\":[\"fixture-a\",\"fixture-b\",\"fixture-c\"],\"label\":\"route-a\",\"providers\":[\"Model A\",\"Model B\",\"Model C\"],\"summary\":\"Route A is preferred.\"}],\"provider_votes\":[{\"position_label\":\"route-a\",\"provider\":\"Model A\"},{\"position_label\":\"route-a\",\"provider\":\"Model B\"},{\"position_label\":\"route-a\",\"provider\":\"Model C\"}],\"recommendation\":\"Use Route A with the stated safeguards.\",\"verdict_applies\":true,\"verdict_type\":\"decision\"}" } } ], @@ -1510,12 +1510,12 @@ } }, { - "request_hash": "sha256:4e9493f40db3dbe48b57ec3df9f941b2444101d9775166b4d353c6d4b9a1c9bc", + "request_hash": "sha256:8a1487cb180ea93400d8353a325c98ea61865d319cace2bd9849e4eaf9347683", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 32, + "max_tokens": 3008, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1543,12 +1543,12 @@ } }, { - "request_hash": "sha256:263812a7f4a8a162ead8f9ea5b142a99b1f931b6beb14c988e0519aa1eab5ae0", + "request_hash": "sha256:55ed4f3ae8cb471849dc7464c677a2eb795e2d94305b625d2b5c8f60f869fe3e", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 32, + "max_tokens": 3136, "messages": [ { "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour draft:\nFictional decision artifact 044.\n\nProduce one complete, improved final answer.", @@ -1576,12 +1576,12 @@ } }, { - "request_hash": "sha256:9a6ce2607e0eb30c7477f008399efde19c5aeed1b7fa1c061620db31b2946fa9", - "occurrence_index": 2, + "request_hash": "sha256:7fe02d317cbc5fd6bc862df80f6418ba7d1cdbd0aac145add051fcd20c6ad575", + "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 786, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1609,12 +1609,12 @@ } }, { - "request_hash": "sha256:f97d7447f59a3a1923b83ecdbd2a256e788aa0306b80e34487c82f33195cc1c8", - "occurrence_index": 2, + "request_hash": "sha256:2e653c544f6ead81a481fad8280b49024ade5526a96780e6af5a76f9cf41c3d3", + "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 786, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1642,12 +1642,12 @@ } }, { - "request_hash": "sha256:8e116cb7c77c18cbe04219e375cd95923cb30625aec753fbc4a4a3ca2907dea3", - "occurrence_index": 2, + "request_hash": "sha256:cc57d8d8ed8963f3537727956243f353427936b20d5365ebb8438e91a93340e1", + "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 786, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1675,12 +1675,12 @@ } }, { - "request_hash": "sha256:5891ce106d20b45e9e85dbe28bfee81eb758e095ea0f9a627e572cf8da21a79f", + "request_hash": "sha256:06f25e59c1322ff7fe6c9c2b228ec2a3301c067441269b7ac01af40b802a0b99", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 914, "messages": [ { "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 046.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 046.\n\n### Model B peer answer\nFictional decision artifact 047.\n\n### Model C peer answer\nFictional decision artifact 048.\n\nProduce one complete revised answer to the original prompt.", @@ -1708,12 +1708,12 @@ } }, { - "request_hash": "sha256:0d7115e520e1bce4daa06cc0a8cf9b25dc5002ee2d60df2dfa00d38ee9cd58c4", + "request_hash": "sha256:040bd4574ded659e69dac9a53f11a59e72a9b79327e3ccc42a494732f0e3ccb8", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 914, "messages": [ { "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 047.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 046.\n\n### Model B peer answer\nFictional decision artifact 047.\n\n### Model C peer answer\nFictional decision artifact 048.\n\nProduce one complete revised answer to the original prompt.", @@ -1741,12 +1741,12 @@ } }, { - "request_hash": "sha256:3fa9111eb2e505f72e5b2b80494e9098f47aab7e444d228f956a54834b76e93f", + "request_hash": "sha256:3fab1afdadb0de5be2874990c6add5e0551b7cbbd12043f3ae2de3d613233996", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 914, "messages": [ { "content": "Original prompt and public material:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nYour original answer:\nFictional decision artifact 048.\n\nAnonymized peer panel:\n\n### Model A peer answer\nFictional decision artifact 046.\n\n### Model B peer answer\nFictional decision artifact 047.\n\n### Model C peer answer\nFictional decision artifact 048.\n\nProduce one complete revised answer to the original prompt.", @@ -1774,12 +1774,12 @@ } }, { - "request_hash": "sha256:4a109d156624f266581a7549ea1fc7661a473c4a0db9fabc2614754a2ac6020f", + "request_hash": "sha256:4c8bb8faba7e3e7bd16446aba2ada2034585c3923da91570f5e599f8c79e4a2b", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 10, + "max_tokens": 1044, "messages": [ { "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", @@ -1811,12 +1811,12 @@ } }, { - "request_hash": "sha256:9a6ce2607e0eb30c7477f008399efde19c5aeed1b7fa1c061620db31b2946fa9", - "occurrence_index": 3, + "request_hash": "sha256:94eb52642fa244a450a865a90d5f84760985a99296505bb7c65d927faeb43f27", + "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1844,12 +1844,12 @@ } }, { - "request_hash": "sha256:f97d7447f59a3a1923b83ecdbd2a256e788aa0306b80e34487c82f33195cc1c8", - "occurrence_index": 3, + "request_hash": "sha256:85c1e9279d0f7807e5b2e11ee8e5b7598b6c0cd3e6902a62af28977b29770f8f", + "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1877,12 +1877,12 @@ } }, { - "request_hash": "sha256:8e116cb7c77c18cbe04219e375cd95923cb30625aec753fbc4a4a3ca2907dea3", - "occurrence_index": 3, + "request_hash": "sha256:d948e25d735ca60fa12fa1f58a1a13a3b83756c89a14cb6aec3f0636dd8c1d79", + "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -1910,12 +1910,12 @@ } }, { - "request_hash": "sha256:75b5a421db80038334f7a5b7e022dac2bc798942ae5fbb5d4f06133aa0c7d82b", + "request_hash": "sha256:4c6ebb9f273a36d41ad3ef449a2651cf6ddf019bec6b78aa216f1e0aa39172cf", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -1947,12 +1947,12 @@ } }, { - "request_hash": "sha256:c9f3331e3a16ca7c13b4a7cab3aceceb523f21b2489c71832187a29c96a6e071", + "request_hash": "sha256:0b2b098a01da3535b62416171121f02fd789679b734ed30a75871e89811d9196", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -1984,12 +1984,12 @@ } }, { - "request_hash": "sha256:d5f31c59028e4f637aff0c1339a5798b39dd5749b87ed5417f68a10a18380e26", + "request_hash": "sha256:f4f04f7567a2f45c8b82f6075e7783cc587f7c421a3c585f1f72e4d4feedd622", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 9, + "max_tokens": 841, "messages": [ { "content": "You are a claim auditor in an elite decision council. Independently stress-test the anonymized answers against the original prompt. Organize your audit into three explicit categories: SUPPORTED claims, CONFLICTING claims, and EXTERNALLY UNVERIFIED claims. Distinguish evidence supplied in the answers from assumptions that would require outside verification. Do not invent citations, sources, quotations, measurements, or facts. The displayed answer IDs provide within-run answer provenance only; they do not prove any claim against external sources. Cite those IDs when identifying a claim. Preserve meaningful minority positions and be precise about uncertainty.", @@ -2021,12 +2021,12 @@ } }, { - "request_hash": "sha256:90717487011b129011d0b7db7fe50382d9967055a302cd0e5324ca0d593e490b", + "request_hash": "sha256:eb274049333444ff45eeb78704ac8ade85da973fc8d5f854e024d053a6ecb4bc", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 10, + "max_tokens": 1098, "messages": [ { "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", @@ -2058,12 +2058,12 @@ } }, { - "request_hash": "sha256:64c25574dc75e5cb8785b3accba9028868bff1e40e1f871b2c7dff341d1958a1", + "request_hash": "sha256:1272f0bc2baa43e27e852787219b6b445ff23481fe9d7785e88c63a72dea8a2a", "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 16, + "max_tokens": 1472, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -2091,12 +2091,12 @@ } }, { - "request_hash": "sha256:cfabe1fcf9f9aa97f5ce3c6f5fd00c0b58e0d19b6328b0858efe8559d145ca8a", + "request_hash": "sha256:3e871bdcc4a67bae1bba8dc450a253797ad6d689350c9cc0ae1bfec5b8e79f58", "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 16, + "max_tokens": 1472, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -2124,12 +2124,12 @@ } }, { - "request_hash": "sha256:66b06f34e69ffb9fa4bfaa79969147bd90b1c424dfd19a950c80c57a953eeb0b", + "request_hash": "sha256:2f49af35b10a45083cec38a6929c4f24840fd9ddb9a0e8337e3e1840ad0e120c", "occurrence_index": 1, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 16, + "max_tokens": 1472, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", @@ -2157,12 +2157,12 @@ } }, { - "request_hash": "sha256:368231f59475acad0fc1c41811ff527bbbe2a94f4acde23932c8d21f178f9c3c", + "request_hash": "sha256:3fab8223b5f796252713e1e765b6b8413b6a612e4655625e09e68a4c29d00cc1", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 16, + "max_tokens": 1728, "messages": [ { "content": "You are the synthesizer of a council of AI models. You are given the same user prompt that was posed to several models, plus each model's answer. Produce one consolidated, accurate answer. Reconcile agreements, surface and adjudicate disagreements, and note any answer that is clearly wrong. Do not invent a model's position; rely only on the answers provided.", @@ -2194,12 +2194,12 @@ } }, { - "request_hash": "sha256:123070814d2b1c5283590f1d4ee4cd72d3cde87a88ac3d5e6cee374f810c612e", + "request_hash": "sha256:9097f135936dad4b6b6b32a146dce9a8bf225304e952fe95d11c0fa5de2a7ce8", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", "body": { - "max_tokens": 64, + "max_tokens": 6144, "messages": [ { "content": "Select the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.", diff --git a/tests/test_verdict_synthesis.py b/tests/test_verdict_synthesis.py index 8a3a1c3..6ebbb46 100644 --- a/tests/test_verdict_synthesis.py +++ b/tests/test_verdict_synthesis.py @@ -27,6 +27,8 @@ import json +import pytest + from conclave import agreement from conclave.manifest import VerdictExtraction from conclave.models import ModelAnswer @@ -513,6 +515,22 @@ def test_repair_once_then_success(monkeypatch): assert result.verdict.consensus_label == "unanimous" +@pytest.mark.asyncio +async def test_extract_verdict_accepts_an_injected_guarded_call_seam() -> None: + fake = _ScriptedExtractor(json.dumps(_payload())) + + result = await extract_verdict( + "Choose A or B.", + [member("alpha", "A", answer_id="alpha-1"), member("beta", "A", answer_id="beta-1")], + synthesizer_name=SYNTH_NAME, + synthesizer_model_id=SYNTH_MODEL_ID, + call_model_func=fake, + ) + + assert result.verdict is not None + assert len(fake.calls) == 1 + + def test_absent_when_extractor_returns_error(monkeypatch, conclave_caplog): """An extractor ModelAnswer with .error (no answer) โ†’ fallback after repair.""" members = [ From d0af2eae3823f2983856a884da7700dd5cf83924 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 22:43:03 -0400 Subject: [PATCH 13/14] fix(evals): authenticate checkpoints and bound dry runs --- CHANGELOG.md | 5 + DOCUMENTATION_INDEX.md | 4 +- README.md | 16 +- SYSTEM_CONTEXT_DIAGRAM.md | 11 +- docs/PRODUCT_DESIGN_DOCUMENT.md | 8 +- ...-07-18-h1-live-evaluation-runner-design.md | 37 ++-- src/conclave/eval_cli.py | 60 +++++- src/conclave/evals/live.py | 202 +++++++++++++++--- src/conclave/evals/pricing.py | 12 +- src/conclave/evals/runner.py | 38 +++- tests/evals/test_eval_cli.py | 99 ++++++++- tests/evals/test_live_checkpoint.py | 106 ++++++++- tests/evals/test_live_estimate.py | 105 ++++++++- tests/evals/test_live_gateway.py | 1 + tests/evals/test_live_replay.py | 3 + tests/evals/test_live_runner.py | 28 ++- tests/evals/test_pricing.py | 39 +++- tests/fixtures/evals/live_smoke/manifest.json | 28 +-- .../fixtures/evals/live_smoke/price_book.json | 9 +- tests/fixtures/evals/live_smoke/replay.json | 2 +- 20 files changed, 703 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c41ddb..c7167f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 reservation before allowing one in-flight provider call, and never repeats an interrupted cell on resume. Its outputs are not decision eligible: the smoke verifies correctness only, not efficiency or decision quality; the offline/open-book 24-task fixture remains separate. +- **Authenticated live checkpoints and bounded UTF-8 estimates.** Paid execution additionally + requires an owner-only `--checkpoint-seal-key-file`; HMAC-SHA256 v2 checkpoints reject old, + forged, or wrong-key state without storing the key. Frozen price entries now bind + `max_output_bytes_per_token`, preventing one-byte dry-run placeholders from understating + multibyte upstream output expansion. ### Fixed diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index 8ef669d..5b0adf2 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -34,7 +34,7 @@ the canonical authority spec on top of those. | **H1 Evaluation Plan** | [`docs/plans/2026-07-17-h1-budget-matched-evaluation.md`](docs/plans/2026-07-17-h1-budget-matched-evaluation.md) | TDD delivery plan for the offline, budget-matched evaluation substrate. | | **H1 Method Hardening** | [`docs/plans/2026-07-17-h1-method-hardening-design.md`](docs/plans/2026-07-17-h1-method-hardening-design.md) | Paid-study provenance, task-clustered inference, grader controls, confirmatory refusal boundaries, and QA design. | | **H1 Synthetic QA Pack** | [`studies/elite_qa_v1/README.md`](studies/elite_qa_v1/README.md) | Balanced 24-task open-book harness fixture, QA protocol, and confirmatory preregistration template. | -| **H1 Live Runner Design** | [`docs/plans/2026-07-18-h1-live-evaluation-runner-design.md`](docs/plans/2026-07-18-h1-live-evaluation-runner-design.md) | Sequential paid-exploratory execution, frozen pricing, pessimistic reservations, USD 10 cap, atomic checkpoints, and no-repeat resume. | +| **H1 Live Runner Design** | [`docs/plans/2026-07-18-h1-live-evaluation-runner-design.md`](docs/plans/2026-07-18-h1-live-evaluation-runner-design.md) | Sequential paid-exploratory execution, hash-bound UTF-8 estimates, USD 10 cap, authenticated checkpoints, and no-repeat resume. | | **H1 Live Runner Plan** | [`docs/plans/2026-07-18-h1-live-evaluation-runner.md`](docs/plans/2026-07-18-h1-live-evaluation-runner.md) | Exact TDD tasks for the six live conditions, dry-run estimator, replay fixtures, CLI gate, and correctness-only paid smoke. | --- @@ -67,7 +67,7 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Models | [`src/conclave/models.py`](src/conclave/models.py) | Stable Pydantic contract, including `EliteResult` phase artifacts and backward-compatible `CouncilResult.elite`. | | CLI | [`src/conclave/cli.py`](src/conclave/cli.py) | Five released `conclave ask` modes plus source-only Elite and `providers`; Elite exits 1 unless decision readiness is `ready`, emits full JSON before failure, and rejects streaming. | | Experimental evals | [`src/conclave/evals/`](src/conclave/evals/) | Versioned DSE-708 planning, strict replay, failure-inclusive running, blinding, scoring, and exploratory reports; no product-quality claim. | -| Eval CLI | [`src/conclave/eval_cli.py`](src/conclave/eval_cli.py) | Offline `plan/run/blind/report` plus `eval live`; live defaults to dry-run and paid execution requires `--execute` with exact USD 10.00 approval. | +| Eval CLI | [`src/conclave/eval_cli.py`](src/conclave/eval_cli.py) | Offline `plan/run/blind/report` plus `eval live`; paid execution requires `--execute`, exact USD 10.00 approval, and an owner-only `--checkpoint-seal-key-file`. | | Logging | [`src/conclave/logging.py`](src/conclave/logging.py) | Logger factory; stderr; verbosity via `CONCLAVE_LOG_LEVEL` (default `WARNING`). | ## Tests diff --git a/README.md b/README.md index 193ce50..4f17540 100644 --- a/README.md +++ b/README.md @@ -152,20 +152,20 @@ is all known providers. ### Experimental evaluation (DSE-708) -Offline `conclave eval run` validates only a frozen replay artifact and never calls a provider. -The live lane is **paid exploratory only**. Dry-run is the default; execution requires both `--execute` and exact -`--approve-spend-usd 10.00`. During execution, one provider call is in flight, a reservation -is persisted before each call, and resume never repeats an interrupted cell. Outputs stay -labelled paid exploratory and not decision eligible. The 24-task fixture remains -offline/open-book and is not the paid smoke corpus. Smoke checks correctness only, not -efficiency or decision quality; no result supports a product-quality or confirmatory claim. +Offline `conclave eval run` validates a frozen replay and never calls a provider. The live lane +is **paid exploratory only**. Dry-run is the default; execution also requires `--execute`, exact +`--approve-spend-usd 10.00`, and an owner-only `--checkpoint-seal-key-file` containing at least +32 random bytes. Checkpoints use a versioned HMAC-SHA256 seal; the key is never serialized. +Each frozen price entry attests `max_output_bytes_per_token`, so estimates bound inserted UTF-8 bytes. One provider call is in flight, a reservation is persisted before each call, and resume never repeats an +interrupted cell. Smoke checks correctness only, not efficiency or decision quality; outputs +remain paid exploratory, not decision eligible, and separate from the offline 24-task fixture. ```bash conclave eval plan path/tasks.json path/manifest.json --study-id qa --replicates 2 --seed 19 --max-output-tokens 1200 conclave eval run path/manifest.json path/run.json --replay-artifact path/replay.json conclave eval live path/manifest.json path/tasks.json path/prices.json path/run.json path/checkpoint.json path/receipts.json conclave eval live path/manifest.json path/tasks.json path/prices.json path/run.json path/checkpoint.json path/receipts.json \ - --execute --approve-spend-usd 10.00 + --execute --approve-spend-usd 10.00 --checkpoint-seal-key-file path/checkpoint-seal.key ``` Add `--stream` to render member (and synthesizer) tokens live as they arrive diff --git a/SYSTEM_CONTEXT_DIAGRAM.md b/SYSTEM_CONTEXT_DIAGRAM.md index a027db4..f67bd5f 100644 --- a/SYSTEM_CONTEXT_DIAGRAM.md +++ b/SYSTEM_CONTEXT_DIAGRAM.md @@ -35,7 +35,7 @@ flowchart TB elite["Elite (UNRELEASED)
initial โ†’ claim audit โ†’ revision
fixed 3-success gate each phase"] evalcli["Experimental eval CLI (DSE-708)
offline replay ยท live dry-run
gated paid exploratory execute"] evalartifacts["Versioned eval artifacts
manifest ยท run ยท grader set + separate map
exploratory report"] - liveguard["Live guard
exact USD 10.00 approval ยท reservation before call
one in flight ยท no-repeat resume"] + liveguard["Live guard
exact USD 10.00 approval ยท owner-only seal key
HMAC checkpoint ยท one in flight ยท no-repeat resume"] registry["Registry ยท name to model-id
key PRESENCE only, never values (registry.py)"] config["Config loader ยท custom endpoints (config.py)"] models["Result contract ยท CouncilResult v2
answers ยท verdict ยท consensus ยท manifest (models.py)"] @@ -133,10 +133,11 @@ flowchart TB orchestrator. There is no behavior in the CLI that the library can't reach. - **The DSE-708 eval lane is paid exploratory only.** Offline `eval run` still validates pre-recorded artifacts without provider calls. `eval live` defaults to dry-run; execution - requires `--execute` plus exact USD 10.00 approval. One provider call is in flight, its - reservation is persisted first, and resume skips an interrupted cell rather than repeating - it. The capped smoke checks correctness only, not efficiency or decision quality, and its - outputs remain not decision eligible. + requires `--execute`, exact USD 10.00 approval, and an owner-only checkpoint-seal key file. + HMAC-SHA256 authenticates checkpoint state, and the price hash includes each model's + `max_output_bytes_per_token` estimate bound. One call is in flight, its reservation persists + first, and resume skips an interrupted cell. The smoke checks correctness onlyโ€”not efficiency + or decision qualityโ€”and remains not decision eligible. - **mcp-warden is dashed and dev-time.** The dotted edge from `mcp-warden` to the library is deliberate: warden imports conclave **only at design/eval time**. conclave is stochastic and must never sit in warden's deterministic runtime decision path. See PDD diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index 1b8d176..bc27b1a 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -413,10 +413,10 @@ source-auditable language is therefore too broad until source grounding ships. E implemented but unreleased in source. H1 also includes an opt-in live runner that is **paid exploratory only**. Dry-run is the default; -paid execution requires `--execute` and exact `--approve-spend-usd 10.00`. One provider call is -in flight, its reservation is persisted before each call, and resume never repeats an interrupted -cell. The 24-task fixture remains offline/open-book and is not the paid smoke corpus. -The smoke establishes correctness only, not efficiency or decision quality; its artifacts remain not decision eligible and cannot change a confirmatory gate. +paid execution requires `--execute`, exact `--approve-spend-usd 10.00`, and an owner-only +`--checkpoint-seal-key-file` with at least 32 random bytes. Versioned HMAC-SHA256 checkpoints +never serialize that key; frozen `max_output_bytes_per_token` attestations bound inserted UTF-8 +bytes. One call is in flight, reservations persist first, and resume never repeats interrupted cells. The smoke proves correctness onlyโ€”not efficiency or decision qualityโ€”and remains not decision eligible. The canonical roadmap is [`docs/plans/2026-07-17-decision-quality-roadmap.md`](plans/2026-07-17-decision-quality-roadmap.md): diff --git a/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md b/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md index 52ab036..5b6e3f0 100644 --- a/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md +++ b/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md @@ -26,8 +26,8 @@ orchestration, call order, reservations, and persistence. - `pricing.py` defines and loads an external, immutable price book. It validates exact provider/model/revision coverage, canonical hashing, USD currency, positive pessimistic - input/output rates, and the `FrozenStudyDesign.price_snapshot` binding. No current prices - are compiled into library code. + input/output rates, a positive operator-attested `max_output_bytes_per_token`, and the + `FrozenStudyDesign.price_snapshot` binding. No current prices are compiled into library code. - `live_protocols.py` defines the six versioned condition call graphs and deterministic per-stage output-token allocation. Calls are awaited serially and all stage caps sum to the cell's frozen `max_output_tokens`. @@ -35,7 +35,8 @@ orchestration, call order, reservations, and persistence. interruption recovery, dry-run estimation, and final `StudyRun` assembly. - `eval_cli.py` adds a separate `conclave eval live` surface. Dry-run is the default. Paid execution requires both `--execute` and an `--approve-spend-usd` value that exactly matches - the frozen manifest ceiling. + the frozen manifest ceiling, plus `--checkpoint-seal-key-file` naming an owner-only POSIX + regular file containing at least 32 operator-generated random bytes. The new path accepts only a complete `paid_exploratory_pilot` manifest with a frozen design. It rejects legacy synthetic manifests and confirmatory manifests. @@ -49,11 +50,14 @@ the snapshot ID, capture time, and currency must also match. Duplicate or missin unknown models, non-USD currency, and nonpositive rates fail before key resolution or network access. -Each call reservation is computed from the frozen call specification, exact public prompt, -fixed prompt-template allowance, upstream stage token ceilings that may be inserted into a -later prompt, provider framing allowance, and the call's output-token ceiling. The price book -uses maximum applicable input and output rates, not discounted or cached rates. Reservation -arithmetic uses `Decimal` and rounds upward to USD microcents. Tests pin the formula. +Execution reservations use the exact UTF-8 byte length of resolved messages plus provider +framing and the call's output-token ceiling; treating every input byte as a possible token is +pessimistic. Dry-run cannot know future upstream text, so it removes its internal sentinels and +reserves each upstream token ceiling multiplied by the maximum +`max_output_bytes_per_token` attestation in the frozen price book. That attestation is part of +the canonical price hash, so drift invalidates the manifest binding. The price book uses +maximum input/output rates, not discounted or cached rates. `Decimal` arithmetic rounds upward +to USD microcents. Tests pin the formula and multibyte maximum-expansion case. Before network I/O: @@ -134,10 +138,13 @@ basis, outcome, and bounded error category. ## Checkpoint and resume -The checkpoint is bound to the manifest hash, price-book hash, public-task hash, and USD -10.00 ceiling. It contains completed `RunRecord` values, completed call receipts, committed -cost, and at most one active cell with at most one pending call. It never stores headers, -credential values, endpoint query secrets, raw exceptions, or grader material. +Checkpoint format `conclave_live_checkpoint_v2` is authenticated with HMAC-SHA256 under the +external seal key. Old unkeyed checkpoints, a wrong key, or a publicly recomputed SHA-256 +digest fail closed. The key is required by the paid `run_live_study` API but is never serialized. +The authenticated payload binds the manifest hash, price-book hash, public-task hash, USD +10.00 ceiling, records, receipts, committed cost, and active/pending state. It never stores +headers, credential values, endpoint query secrets, raw exceptions, grader material, or the +seal key. Every transition is written to a temporary file in the destination directory, flushed, `fsync`ed, and installed with `os.replace`. Before replacement, serialized content must be @@ -157,7 +164,7 @@ non-executed cell remains in the denominator. ## Dry-run and replay Dry-run traverses the exact same condition call graph without loading configuration, reading -keys, or invoking transport. It reports planned cells, maximum provider calls, worst-case +provider or checkpoint-seal keys, or invoking transport. It reports planned cells, maximum provider calls, worst-case reserved cost by roster and condition, largest single reservation, total upper bound, the USD 10.00 ceiling, and whether the complete plan fits. It is an authorization aid, not a bill forecast. @@ -173,6 +180,8 @@ asserts exact consumption, and produces the same call receipts and `StudyRun` on |---|---| | Snapshot, manifest, task, or checkpoint hash drift | Abort before keys or network. | | Missing/duplicate/unknown price entry | Abort before keys or network. | +| Missing/short/insecure seal-key file or unsupported POSIX checks | Abort before live execution. | +| Old, forged, or wrong-key checkpoint | Reject before state or cost is trusted. | | Next reservation crosses USD 10.00 | No call; mark remaining cells `budget_exhausted`. | | Provider timeout/error/malformed output | Bounded failure receipt; retain cell in denominator. | | Missing usage | Charge full reservation; mark cost basis pessimistic. | @@ -191,6 +200,8 @@ asserts exact consumption, and produces the same call receipts and `StudyRun` on - Total charged cost never exceeds the manifest-bound USD 10.00 ceiling in success, timeout, missing-usage, budget-exhaustion, and crash/resume tests. - Resume never repeats an interrupted paid cell. +- Tests reject public resealing, wrong keys, old formats, and group/other-readable key files. +- Maximum-expansion multibyte execution reservations do not exceed the dry-run estimate. - Every planned cell appears once in the final `StudyRun`, including unscheduled failures. - Live/checkpoint/replay artifacts contain no credential values or raw exception chains. - Focused tests, full pytest, Ruff lint/format, `git diff --check`, and Gitleaks pass. diff --git a/src/conclave/eval_cli.py b/src/conclave/eval_cli.py index 1600fbd..4410c76 100644 --- a/src/conclave/eval_cli.py +++ b/src/conclave/eval_cli.py @@ -5,6 +5,7 @@ import asyncio import json import os +import stat import tempfile from collections import Counter from decimal import Decimal, InvalidOperation @@ -125,6 +126,37 @@ def _require_live_approval(value: str | None, manifest: StudyManifest) -> None: _abort("spend approval must exactly match USD 10.00 and the frozen design ceiling") +def _read_checkpoint_seal_key(path: Path | None) -> bytes: + if path is None: + _abort("--execute requires --checkpoint-seal-key-file") + if os.name != "posix" or not hasattr(os, "O_NOFOLLOW"): + _abort("checkpoint seal key permission checks are unsupported on this platform") + + descriptor: int | None = None + try: + flags = os.O_RDONLY | os.O_NOFOLLOW + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + descriptor = os.open(path, flags) + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode): + _abort("checkpoint seal key must be a regular POSIX file") + if stat.S_IMODE(file_stat.st_mode) & (stat.S_IRWXG | stat.S_IRWXO): + _abort("checkpoint seal key file must be owner-only") + chunks = [] + while chunk := os.read(descriptor, 4096): + chunks.append(chunk) + seal_key = b"".join(chunks) + except OSError as exc: + _abort(f"could not securely read checkpoint seal key file: {exc.strerror or 'I/O error'}") + finally: + if descriptor is not None: + os.close(descriptor) + if len(seal_key) < 32: + _abort("checkpoint seal key file must contain at least 32 bytes") + return seal_key + + @app.command("live") def live_command( manifest_path: Annotated[Path, typer.Argument(exists=True, readable=True)], @@ -147,6 +179,16 @@ def live_command( help="Allow the guarded live runner to reach providers.", ), ] = False, + checkpoint_seal_key_file: Annotated[ + Path | None, + typer.Option( + "--checkpoint-seal-key-file", + help=( + "Owner-only POSIX file containing at least 32 random bytes; " + "required only with --execute." + ), + ), + ] = None, ) -> None: """Estimate by default; execute only a frozen USD 10 paid exploratory pilot.""" @@ -170,8 +212,20 @@ def live_command( return _require_live_approval(approve_spend_usd, manifest) - if len({output.resolve(), checkpoint.resolve(), receipts.resolve()}) != 3: - _abort("run, checkpoint, and receipt artifacts require separate paths") + checkpoint_seal_key = _read_checkpoint_seal_key(checkpoint_seal_key_file) + assert checkpoint_seal_key_file is not None # narrowed by the fail-closed loader + if ( + len( + { + output.resolve(), + checkpoint.resolve(), + receipts.resolve(), + checkpoint_seal_key_file.resolve(), + } + ) + != 4 + ): + _abort("run, checkpoint, receipt, and seal-key paths must be separate") try: checkpoint.parent.mkdir(parents=True, exist_ok=True) study_run = asyncio.run( @@ -180,6 +234,7 @@ def live_command( tasks=tasks, price_book=price_book, checkpoint_path=checkpoint, + checkpoint_seal_key=checkpoint_seal_key, ) ) bindings = build_checkpoint_bindings( @@ -190,6 +245,7 @@ def live_command( final_checkpoint = load_live_checkpoint( checkpoint, expected_bindings=bindings, + seal_key=checkpoint_seal_key, ) except (OSError, LiveGatewayError, RunValidationError, ValidationError, ValueError) as exc: _abort(f"live execution rejected: {exc}") diff --git a/src/conclave/evals/live.py b/src/conclave/evals/live.py index 3c43ac8..da195a7 100644 --- a/src/conclave/evals/live.py +++ b/src/conclave/evals/live.py @@ -4,11 +4,12 @@ import asyncio import hashlib +import hmac import inspect import json import os import tempfile -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from decimal import ROUND_CEILING, Decimal, localcontext from pathlib import Path from typing import Literal @@ -55,6 +56,20 @@ _BOUNDED_PROVIDER_ERROR_CATEGORIES = frozenset( {"authentication", "rate_limit", "timeout", "transport", "provider_error"} ) +CHECKPOINT_FORMAT_VERSION = "conclave_live_checkpoint_v2" +_MINIMUM_CHECKPOINT_SEAL_KEY_BYTES = 32 +_CHECKPOINT_DECIMAL_FIELDS = frozenset( + { + "hard_cap_usd", + "committed_cost_usd", + "input_ceiling_usd_per_million_tokens", + "output_ceiling_usd_per_million_tokens", + "input_cost_upper_bound_usd", + "output_cost_upper_bound_usd", + "reserved_cost_usd", + "charged_cost_usd", + } +) class LiveGatewayError(RuntimeError): @@ -173,21 +188,33 @@ def _canonical_hash(payload: object) -> str: return f"sha256:{hashlib.sha256(canonical).hexdigest()}" +def _canonical_checkpoint_value(value: object) -> object: + if isinstance(value, dict): + normalized = {} + for key, item in value.items(): + if key in _CHECKPOINT_DECIMAL_FIELDS and isinstance(item, str): + item = Decimal(item) + normalized[str(key)] = _canonical_checkpoint_value(item) + return normalized + if isinstance(value, (list, tuple)): + return [_canonical_checkpoint_value(item) for item in value] + return _canonical_value(value) + + class LiveCheckpoint(EvalModel): """Secret-free, integrity-sealed state for one sequential live study.""" + checkpoint_format_version: Literal["conclave_live_checkpoint_v2"] bindings: CheckpointBindings committed_cost_usd: Decimal = Field(ge=0) hard_cap_breached: bool = False records: tuple[RunRecord, ...] = () receipts: tuple[ProviderCallReceipt, ...] = () active_cell: ActiveCell | None = None - checkpoint_hash: Sha256Digest + checkpoint_hash: str = Field(pattern=r"^hmac-sha256:[0-9a-f]{64}$") @model_validator(mode="after") - def validate_integrity(self) -> LiveCheckpoint: - if self.checkpoint_hash != hash_live_checkpoint(self): - raise ValueError("checkpoint integrity hash mismatch") + def validate_state(self) -> LiveCheckpoint: record_ids = [record.planned_run_id for record in self.records] if len(set(record_ids)) != len(record_ids): raise ValueError("checkpoint records must have unique planned_run_id values") @@ -233,14 +260,47 @@ def build_checkpoint_bindings( ) -def hash_live_checkpoint(checkpoint: LiveCheckpoint) -> str: - """Return a canonical digest over checkpoint content and all study bindings.""" +def _validate_checkpoint_seal_key(seal_key: bytes) -> None: + if not isinstance(seal_key, bytes): + raise CheckpointValidationError("checkpoint seal key must be bytes") + if len(seal_key) < _MINIMUM_CHECKPOINT_SEAL_KEY_BYTES: + raise CheckpointValidationError("checkpoint seal key must contain at least 32 bytes") + + +def _checkpoint_hmac(checkpoint: LiveCheckpoint, *, seal_key: bytes) -> str: + return _checkpoint_payload_hmac( + checkpoint.model_dump(mode="python", exclude={"checkpoint_hash"}), + seal_key=seal_key, + ) + + +def _checkpoint_payload_hmac(payload: Mapping[str, object], *, seal_key: bytes) -> str: + _validate_checkpoint_seal_key(seal_key) + canonical = json.dumps( + _canonical_checkpoint_value(payload), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + digest = hmac.new(seal_key, canonical, hashlib.sha256).hexdigest() + return f"hmac-sha256:{digest}" + + +def hash_live_checkpoint(checkpoint: LiveCheckpoint, *, seal_key: bytes) -> str: + """Return a keyed HMAC over checkpoint content and all study bindings.""" + + return _checkpoint_hmac(checkpoint, seal_key=seal_key) - return _canonical_hash(checkpoint.model_dump(mode="python", exclude={"checkpoint_hash"})) + +def _verify_checkpoint_seal(checkpoint: LiveCheckpoint, *, seal_key: bytes) -> None: + expected = _checkpoint_hmac(checkpoint, seal_key=seal_key) + if not hmac.compare_digest(checkpoint.checkpoint_hash, expected): + raise CheckpointValidationError("integrity live checkpoint") def create_live_checkpoint( *, + seal_key: bytes, bindings: CheckpointBindings, records: tuple[RunRecord, ...] = (), receipts: tuple[ProviderCallReceipt, ...] = (), @@ -253,17 +313,20 @@ def create_live_checkpoint( committed_cost_usd = sum((receipt.charged_cost_usd for receipt in receipts), Decimal("0")) unsealed = LiveCheckpoint.model_construct( schema_version=EVAL_SCHEMA_VERSION, + checkpoint_format_version=CHECKPOINT_FORMAT_VERSION, bindings=bindings, committed_cost_usd=committed_cost_usd, hard_cap_breached=committed_cost_usd > bindings.hard_cap_usd, records=tuple(records), receipts=tuple(receipts), active_cell=active_cell, - checkpoint_hash="sha256:" + ("0" * 64), + checkpoint_hash="hmac-sha256:" + ("0" * 64), ) payload = unsealed.model_dump(mode="python") - payload["checkpoint_hash"] = hash_live_checkpoint(unsealed) - return LiveCheckpoint.model_validate(payload) + payload["checkpoint_hash"] = hash_live_checkpoint(unsealed, seal_key=seal_key) + checkpoint = LiveCheckpoint.model_validate(payload) + _verify_checkpoint_seal(checkpoint, seal_key=seal_key) + return checkpoint def _active_provider_key_values() -> tuple[str, ...]: @@ -276,9 +339,10 @@ def _active_provider_key_values() -> tuple[str, ...]: return tuple(dict.fromkeys(values)) -def _checkpoint_json(checkpoint: LiveCheckpoint) -> str: +def _checkpoint_json(checkpoint: LiveCheckpoint, *, seal_key: bytes) -> str: try: validated = LiveCheckpoint.model_validate(checkpoint.model_dump(mode="python")) + _verify_checkpoint_seal(validated, seal_key=seal_key) except (ValidationError, ValueError) as exc: raise CheckpointValidationError("invalid live checkpoint state") from exc return ( @@ -292,11 +356,16 @@ def _checkpoint_json(checkpoint: LiveCheckpoint) -> str: ) -def write_live_checkpoint(path: str | Path, checkpoint: LiveCheckpoint) -> None: +def write_live_checkpoint( + path: str | Path, + checkpoint: LiveCheckpoint, + *, + seal_key: bytes, +) -> None: """Secret-scan and atomically replace a checkpoint in its destination directory.""" destination = Path(path) - payload = _checkpoint_json(checkpoint) + payload = _checkpoint_json(checkpoint, seal_key=seal_key) if redact(payload) != payload or any( key_value in payload for key_value in _active_provider_key_values() ): @@ -330,17 +399,46 @@ def load_live_checkpoint( path: str | Path, *, expected_bindings: CheckpointBindings, + seal_key: bytes, ) -> LiveCheckpoint: """Strictly load and bind an integrity-sealed checkpoint, failing closed.""" try: with Path(path).open(encoding="utf-8") as handle: - payload = json.load(handle, parse_float=Decimal) + payload = json.load(handle) + if payload.get("checkpoint_format_version") != CHECKPOINT_FORMAT_VERSION: + raise CheckpointValidationError("invalid live checkpoint format version") + expected_fields = { + "schema_version", + "checkpoint_format_version", + "bindings", + "committed_cost_usd", + "hard_cap_breached", + "records", + "receipts", + "active_cell", + "checkpoint_hash", + } + if set(payload) != expected_fields: + raise CheckpointValidationError("invalid live checkpoint fields") + checkpoint_hash = payload["checkpoint_hash"] + unsigned_payload = dict(payload) + unsigned_payload.pop("checkpoint_hash") + expected_hash = _checkpoint_payload_hmac(unsigned_payload, seal_key=seal_key) + if not isinstance(checkpoint_hash, str) or not hmac.compare_digest( + checkpoint_hash, + expected_hash, + ): + raise CheckpointValidationError("integrity live checkpoint") checkpoint = LiveCheckpoint.model_validate(payload) + _verify_checkpoint_seal(checkpoint, seal_key=seal_key) + except CheckpointValidationError: + raise except json.JSONDecodeError as exc: raise CheckpointValidationError("invalid live checkpoint JSON") from exc except (OSError, TypeError, ValidationError, ValueError) as exc: - label = "integrity" if "integrity" in str(exc).lower() else "invalid" + text = str(exc).lower() + label = "integrity" if "integrity" in text or "checkpoint_hash" in text else "invalid" raise CheckpointValidationError(f"{label} live checkpoint") from exc for field in ( @@ -354,7 +452,12 @@ def load_live_checkpoint( return checkpoint -def start_active_cell(checkpoint: LiveCheckpoint, *, planned_run_id: str) -> LiveCheckpoint: +def start_active_cell( + checkpoint: LiveCheckpoint, + *, + planned_run_id: str, + seal_key: bytes, +) -> LiveCheckpoint: """Persist the start of a cell before its first provider reservation.""" if checkpoint.active_cell is not None: @@ -362,6 +465,7 @@ def start_active_cell(checkpoint: LiveCheckpoint, *, planned_run_id: str) -> Liv if not checkpoint.should_execute(planned_run_id): raise CheckpointValidationError("planned run already has a final record") return create_live_checkpoint( + seal_key=seal_key, bindings=checkpoint.bindings, records=checkpoint.records, receipts=checkpoint.receipts, @@ -376,6 +480,7 @@ def start_active_cell(checkpoint: LiveCheckpoint, *, planned_run_id: str) -> Liv def update_live_checkpoint( checkpoint: LiveCheckpoint, *, + seal_key: bytes, pending_call: PendingCall | None, receipts: tuple[ProviderCallReceipt, ...], ) -> LiveCheckpoint: @@ -391,6 +496,7 @@ def update_live_checkpoint( receipt_tuple = tuple(receipts) committed = sum((receipt.charged_cost_usd for receipt in receipt_tuple), Decimal("0")) return create_live_checkpoint( + seal_key=seal_key, bindings=checkpoint.bindings, records=checkpoint.records, receipts=receipt_tuple, @@ -421,7 +527,11 @@ def _interrupted_receipt(pending: PendingCall) -> ProviderCallReceipt: ) -def recover_interrupted_checkpoint(checkpoint: LiveCheckpoint) -> LiveCheckpoint: +def recover_interrupted_checkpoint( + checkpoint: LiveCheckpoint, + *, + seal_key: bytes, +) -> LiveCheckpoint: """Charge pending work and finalize the interrupted cell without retrying it.""" active = checkpoint.active_cell @@ -443,6 +553,7 @@ def recover_interrupted_checkpoint(checkpoint: LiveCheckpoint) -> LiveCheckpoint deviation_codes=("interrupted_cell_not_retried",), ) return create_live_checkpoint( + seal_key=seal_key, bindings=checkpoint.bindings, records=(*checkpoint.records, record), receipts=receipts, @@ -450,7 +561,12 @@ def recover_interrupted_checkpoint(checkpoint: LiveCheckpoint) -> LiveCheckpoint ) -def finish_active_cell(checkpoint: LiveCheckpoint, *, record: RunRecord) -> LiveCheckpoint: +def finish_active_cell( + checkpoint: LiveCheckpoint, + *, + record: RunRecord, + seal_key: bytes, +) -> LiveCheckpoint: """Finalize the active cell while preserving all receipt and cost history.""" active = checkpoint.active_cell @@ -461,6 +577,7 @@ def finish_active_cell(checkpoint: LiveCheckpoint, *, record: RunRecord) -> Live if active.pending_call is not None: raise CheckpointValidationError("cannot finalize a cell with a pending provider call") return create_live_checkpoint( + seal_key=seal_key, bindings=checkpoint.bindings, records=(*checkpoint.records, record), receipts=checkpoint.receipts, @@ -497,13 +614,25 @@ def _resolve_call_price(price_book: PriceBook, call: StageCall) -> ModelPrice: ) -def _reserve_stage_call(call: StageCall, price: ModelPrice) -> CallReservation: +def _reserve_stage_call( + call: StageCall, + price: ModelPrice, + *, + prompt_token_upper_bound: int | None = None, + upstream_output_token_ceilings: Sequence[int] = (), + upstream_output_bytes_per_token: int | None = None, +) -> CallReservation: + if prompt_token_upper_bound is None: + prompt_token_upper_bound = _prompt_token_upper_bound(call) + if upstream_output_bytes_per_token is None: + upstream_output_bytes_per_token = price.max_output_bytes_per_token return reserve_call_cost( price, - prompt_token_upper_bound=_prompt_token_upper_bound(call), + prompt_token_upper_bound=prompt_token_upper_bound, prompt_template_token_allowance=0, provider_framing_token_allowance=_provider_framing_token_allowance(call), - upstream_output_token_ceilings=call.upstream_output_token_ceilings, + upstream_output_token_ceilings=upstream_output_token_ceilings, + upstream_output_bytes_per_token=upstream_output_bytes_per_token, max_output_tokens=call.max_output_tokens, ) @@ -759,23 +888,38 @@ class _LiveEstimateClient: def __init__(self, price_book: PriceBook) -> None: self._price_book = price_book + self._max_output_bytes_per_token = max( + price.max_output_bytes_per_token for price in price_book.entries + ) + self._upstream_placeholders: dict[str, int] = {} self.reservations: list[CallReservation] = [] async def call(self, call: StageCall) -> ModelAnswer: price = _resolve_call_price(self._price_book, call) - self.reservations.append(_reserve_stage_call(call, price)) - if call.stage == "verdict": - return ModelAnswer( - name=call.provider_id, - model_id=call.model_id, - answer="invalid-first-verdict-for-max-graph-estimate", + static_prompt_bytes = _prompt_token_upper_bound(call) + upstream_ceilings: list[int] = [] + for placeholder, token_ceiling in self._upstream_placeholders.items(): + occurrences = sum(message.content.count(placeholder) for message in call.messages) + if occurrences: + static_prompt_bytes -= occurrences * len(placeholder.encode("utf-8")) + upstream_ceilings.extend([token_ceiling] * occurrences) + self.reservations.append( + _reserve_stage_call( + call, + price, + prompt_token_upper_bound=static_prompt_bytes, + upstream_output_token_ceilings=upstream_ceilings, + upstream_output_bytes_per_token=self._max_output_bytes_per_token, ) + ) if call.stage == "verdict_repair": raise _EstimateMaxGraphComplete + placeholder = f"__conclave_estimated_output_{len(self._upstream_placeholders):08d}__" + self._upstream_placeholders[placeholder] = call.max_output_tokens return ModelAnswer( name=call.provider_id, model_id=call.model_id, - answer="x" * call.max_output_tokens, + answer=placeholder, ) diff --git a/src/conclave/evals/pricing.py b/src/conclave/evals/pricing.py index de3c72c..ac06683 100644 --- a/src/conclave/evals/pricing.py +++ b/src/conclave/evals/pricing.py @@ -27,6 +27,7 @@ class ModelPrice(EvalModel): model_revision: str = Field(min_length=1) input_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) output_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) + max_output_bytes_per_token: int = Field(gt=0, strict=True) @field_validator( "input_ceiling_usd_per_million_tokens", @@ -70,6 +71,7 @@ class CallReservation(EvalModel): prompt_template_token_allowance: int = Field(ge=0) provider_framing_token_allowance: int = Field(ge=0) upstream_output_token_ceilings: tuple[int, ...] + upstream_output_bytes_per_token: int = Field(gt=0) input_token_upper_bound: int = Field(ge=0) output_token_upper_bound: int = Field(gt=0) input_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) @@ -110,6 +112,7 @@ def _canonical_entry(entry: ModelPrice) -> dict[str, str]: "output_ceiling_usd_per_million_tokens": _canonical_decimal( entry.output_ceiling_usd_per_million_tokens ), + "max_output_bytes_per_token": str(entry.max_output_bytes_per_token), } @@ -190,6 +193,7 @@ def reserve_call_cost( prompt_template_token_allowance: int, provider_framing_token_allowance: int, upstream_output_token_ceilings: Sequence[int], + upstream_output_bytes_per_token: int, max_output_tokens: int, ) -> CallReservation: """Reserve the pessimistic USD cost of all possible input and output tokens.""" @@ -197,6 +201,11 @@ def reserve_call_cost( _validate_token_bound("prompt_token_upper_bound", prompt_token_upper_bound) _validate_token_bound("prompt_template_token_allowance", prompt_template_token_allowance) _validate_token_bound("provider_framing_token_allowance", provider_framing_token_allowance) + _validate_token_bound( + "upstream_output_bytes_per_token", + upstream_output_bytes_per_token, + positive=True, + ) _validate_token_bound("max_output_tokens", max_output_tokens, positive=True) upstream_ceilings = tuple(upstream_output_token_ceilings) for index, ceiling in enumerate(upstream_ceilings): @@ -206,7 +215,7 @@ def reserve_call_cost( prompt_token_upper_bound + prompt_template_token_allowance + provider_framing_token_allowance - + sum(upstream_ceilings) + + (sum(upstream_ceilings) * upstream_output_bytes_per_token) ) precision = max( 64, @@ -239,6 +248,7 @@ def reserve_call_cost( prompt_template_token_allowance=prompt_template_token_allowance, provider_framing_token_allowance=provider_framing_token_allowance, upstream_output_token_ceilings=upstream_ceilings, + upstream_output_bytes_per_token=upstream_output_bytes_per_token, input_token_upper_bound=input_token_upper_bound, output_token_upper_bound=max_output_tokens, input_ceiling_usd_per_million_tokens=price.input_ceiling_usd_per_million_tokens, diff --git a/src/conclave/evals/runner.py b/src/conclave/evals/runner.py index d3a7f23..35a6859 100644 --- a/src/conclave/evals/runner.py +++ b/src/conclave/evals/runner.py @@ -320,6 +320,7 @@ def _load_or_create_live_checkpoint( *, manifest: StudyManifest, price_book: PriceBook, + seal_key: bytes, ) -> LiveCheckpoint: bindings = build_checkpoint_bindings( manifest, @@ -327,10 +328,14 @@ def _load_or_create_live_checkpoint( hard_cap_usd=LIVE_HARD_CAP_USD, ) if checkpoint_path.exists(): - checkpoint = load_live_checkpoint(checkpoint_path, expected_bindings=bindings) + checkpoint = load_live_checkpoint( + checkpoint_path, + expected_bindings=bindings, + seal_key=seal_key, + ) else: - checkpoint = create_live_checkpoint(bindings=bindings) - write_live_checkpoint(checkpoint_path, checkpoint) + checkpoint = create_live_checkpoint(bindings=bindings, seal_key=seal_key) + write_live_checkpoint(checkpoint_path, checkpoint, seal_key=seal_key) planned_ids = {planned_run.planned_run_id for planned_run in manifest.planned_runs} record_ids = {record.planned_run_id for record in checkpoint.records} @@ -340,8 +345,8 @@ def _load_or_create_live_checkpoint( ): raise CheckpointValidationError("checkpoint contains an unplanned live cell") if checkpoint.active_cell is not None: - checkpoint = recover_interrupted_checkpoint(checkpoint) - write_live_checkpoint(checkpoint_path, checkpoint) + checkpoint = recover_interrupted_checkpoint(checkpoint, seal_key=seal_key) + write_live_checkpoint(checkpoint_path, checkpoint, seal_key=seal_key) return checkpoint @@ -410,6 +415,7 @@ def _materialize_remaining( planned_runs: Sequence[PlannedRun], *, error_category: str, + seal_key: bytes, ) -> LiveCheckpoint: recorded = {record.planned_run_id for record in checkpoint.records} additions = tuple( @@ -424,6 +430,7 @@ def _materialize_remaining( if planned_run.planned_run_id not in recorded ) return create_live_checkpoint( + seal_key=seal_key, bindings=checkpoint.bindings, records=(*checkpoint.records, *additions), receipts=checkpoint.receipts, @@ -453,6 +460,7 @@ async def run_live_study( tasks: Sequence[PublicTask], price_book: PriceBook, checkpoint_path: str | Path, + checkpoint_seal_key: bytes, call_model_func: CallModel = call_model, ) -> StudyRun: """Execute a frozen paid-exploratory matrix under the exact USD 10.00 cap.""" @@ -466,6 +474,7 @@ async def run_live_study( path=path, task_by_id=task_by_id, roster_by_id=roster_by_id, + checkpoint_seal_key=checkpoint_seal_key, call_model_func=call_model_func, ) @@ -477,6 +486,7 @@ async def _run_live_study_with_lease( path: Path, task_by_id: Mapping[str, PublicTask], roster_by_id: Mapping[str, tuple], + checkpoint_seal_key: bytes, call_model_func: CallModel, ) -> StudyRun: """Run one validated live study while its process lease is held.""" @@ -485,6 +495,7 @@ async def _run_live_study_with_lease( path, manifest=manifest, price_book=price_book, + seal_key=checkpoint_seal_key, ) def persist( @@ -494,10 +505,11 @@ def persist( nonlocal checkpoint checkpoint = update_live_checkpoint( checkpoint, + seal_key=checkpoint_seal_key, pending_call=pending_call, receipts=receipts, ) - write_live_checkpoint(path, checkpoint) + write_live_checkpoint(path, checkpoint, seal_key=checkpoint_seal_key) client = LiveProviderClient( price_book=price_book, @@ -515,8 +527,9 @@ def persist( checkpoint = start_active_cell( checkpoint, planned_run_id=planned_run.planned_run_id, + seal_key=checkpoint_seal_key, ) - write_live_checkpoint(path, checkpoint) + write_live_checkpoint(path, checkpoint, seal_key=checkpoint_seal_key) receipt_start_index = checkpoint.active_cell.receipt_start_index started = time.perf_counter() try: @@ -543,13 +556,15 @@ def persist( checkpoint = finish_active_cell( checkpoint, record=_record_from_execution(planned_run.planned_run_id, execution), + seal_key=checkpoint_seal_key, ) checkpoint = _materialize_remaining( checkpoint, planned_runs[index + 1 :], error_category="budget_exhausted", + seal_key=checkpoint_seal_key, ) - write_live_checkpoint(path, checkpoint) + write_live_checkpoint(path, checkpoint, seal_key=checkpoint_seal_key) break except (ReservationBreachError, GatewayStoppedError) as exc: receipts = _cell_receipts(checkpoint, receipt_start_index=receipt_start_index) @@ -568,13 +583,15 @@ def persist( checkpoint = finish_active_cell( checkpoint, record=_record_from_execution(planned_run.planned_run_id, execution), + seal_key=checkpoint_seal_key, ) checkpoint = _materialize_remaining( checkpoint, planned_runs[index + 1 :], error_category="gateway_stopped", + seal_key=checkpoint_seal_key, ) - write_live_checkpoint(path, checkpoint) + write_live_checkpoint(path, checkpoint, seal_key=checkpoint_seal_key) break except Exception: # noqa: BLE001 -- bounded receipts determine the public failure receipts = _cell_receipts(checkpoint, receipt_start_index=receipt_start_index) @@ -591,7 +608,8 @@ def persist( checkpoint = finish_active_cell( checkpoint, record=_record_from_execution(planned_run.planned_run_id, execution), + seal_key=checkpoint_seal_key, ) - write_live_checkpoint(path, checkpoint) + write_live_checkpoint(path, checkpoint, seal_key=checkpoint_seal_key) return _build_live_study_run(manifest, checkpoint) diff --git a/tests/evals/test_eval_cli.py b/tests/evals/test_eval_cli.py index ee00159..ad22df1 100644 --- a/tests/evals/test_eval_cli.py +++ b/tests/evals/test_eval_cli.py @@ -24,12 +24,19 @@ runner = CliRunner() ROOT = Path(__file__).resolve().parents[2] +CHECKPOINT_SEAL_KEY = bytes(range(32)) def _write_json(path, payload) -> None: path.write_text(json.dumps(payload), encoding="utf-8") +def _write_seal_key(path: Path) -> Path: + path.write_bytes(CHECKPOINT_SEAL_KEY) + path.chmod(0o600) + return path + + def _study_artifacts(tmp_path): tasks = [PublicTask(task_id="task-a", prompt="Choose one.")] tasks_path = tmp_path / "tasks.json" @@ -212,6 +219,80 @@ async def forbidden_live_runner(**kwargs): assert provider_calls == 0 +def test_eval_live_execute_requires_owner_only_checkpoint_seal_key_file( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, _, _, tasks_path, manifest_path, price_book_path = _live_artifacts(tmp_path) + runner_calls = 0 + + async def forbidden_live_runner(**kwargs): + nonlocal runner_calls + del kwargs + runner_calls += 1 + raise AssertionError("seal-key validation must happen before live execution") + + monkeypatch.setattr(eval_cli_module, "run_live_study", forbidden_live_runner, raising=False) + base_paths = ( + tmp_path / "run.json", + tmp_path / "checkpoint.json", + tmp_path / "receipts.json", + ) + missing = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + *base_paths, + "--execute", + "--approve-spend-usd", + "10.00", + ), + ) + assert missing.exit_code == 2 + assert "--checkpoint-seal-key-file" in missing.output + + key_path = tmp_path / "checkpoint-seal.key" + key_path.write_bytes(bytes(range(32))) + key_path.chmod(0o640) + insecure = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + *base_paths, + "--execute", + "--approve-spend-usd", + "10.00", + "--checkpoint-seal-key-file", + str(key_path), + ), + ) + assert insecure.exit_code == 2 + assert "owner-only" in insecure.output.lower() + assert runner_calls == 0 + + +def test_eval_live_dry_run_does_not_read_checkpoint_seal_key_file(tmp_path) -> None: + _, _, _, tasks_path, manifest_path, price_book_path = _live_artifacts(tmp_path) + result = runner.invoke( + app, + _live_command( + manifest_path, + tasks_path, + price_book_path, + tmp_path / "run.json", + tmp_path / "checkpoint.json", + tmp_path / "receipts.json", + "--checkpoint-seal-key-file", + str(tmp_path / "does-not-exist.key"), + ), + ) + + assert result.exit_code == 0, result.output + + def test_live_evaluation_docs_preserve_operator_and_claim_boundaries() -> None: docs = { path: " ".join((ROOT / path).read_text(encoding="utf-8").lower().split()) @@ -230,6 +311,10 @@ def test_live_evaluation_docs_preserve_operator_and_claim_boundaries() -> None: assert "dry-run is the default" in corpus assert "--execute" in corpus assert "--approve-spend-usd 10.00" in corpus + assert "--checkpoint-seal-key-file" in corpus + assert "owner-only" in corpus + assert "hmac-sha256" in corpus + assert "max_output_bytes_per_token" in corpus assert "one provider call is in flight" in corpus assert "reservation is persisted before each call" in corpus assert "resume never repeats an interrupted cell" in corpus @@ -304,6 +389,7 @@ def test_eval_live_writes_checkpoint_receipts_and_study_run_atomically( output_path = tmp_path / "artifacts" / "run.json" checkpoint_path = tmp_path / "state" / "checkpoint.json" receipts_path = tmp_path / "artifacts" / "receipts.json" + seal_key_path = _write_seal_key(tmp_path / "checkpoint-seal.key") atomic_paths = [] real_atomic_json = eval_cli_module._atomic_json @@ -332,6 +418,8 @@ def recording_atomic_json(path, value): "--execute", "--approve-spend-usd", "10.00", + "--checkpoint-seal-key-file", + str(seal_key_path), ), ) @@ -353,6 +441,7 @@ def test_eval_live_resume_uses_existing_checkpoint( output_path = tmp_path / "run.json" checkpoint_path = tmp_path / "checkpoint.json" receipts_path = tmp_path / "receipts.json" + seal_key_path = _write_seal_key(tmp_path / "checkpoint-seal.key") first_run = manifest.planned_runs[0] preserved = RunRecord( planned_run_id=first_run.planned_run_id, @@ -367,7 +456,12 @@ def test_eval_live_resume_uses_existing_checkpoint( ) write_live_checkpoint( checkpoint_path, - create_live_checkpoint(bindings=bindings, records=(preserved,)), + create_live_checkpoint( + bindings=bindings, + records=(preserved,), + seal_key=CHECKPOINT_SEAL_KEY, + ), + seal_key=CHECKPOINT_SEAL_KEY, ) provider_calls = 0 @@ -394,6 +488,8 @@ async def injected_runner(**kwargs): "--execute", "--approve-spend-usd", "10.00", + "--checkpoint-seal-key-file", + str(seal_key_path), ), ) @@ -408,6 +504,7 @@ async def injected_runner(**kwargs): resumed_checkpoint = load_live_checkpoint( checkpoint_path, expected_bindings=bindings, + seal_key=CHECKPOINT_SEAL_KEY, ) assert resumed_checkpoint.records[0] == preserved diff --git a/tests/evals/test_live_checkpoint.py b/tests/evals/test_live_checkpoint.py index 3e51532..41a9d78 100644 --- a/tests/evals/test_live_checkpoint.py +++ b/tests/evals/test_live_checkpoint.py @@ -19,13 +19,6 @@ ProviderCallCostBasis, ProviderCallReceipt, ReservationBreachError, - create_live_checkpoint, - finish_active_cell, - load_live_checkpoint, - recover_interrupted_checkpoint, - start_active_cell, - update_live_checkpoint, - write_live_checkpoint, ) from conclave.evals.live_protocols import ChatMessage, StageCall from conclave.evals.models import RunRecord @@ -37,6 +30,43 @@ DIGEST_C = "sha256:" + "c" * 64 RUN_A = "run_" + "a" * 24 RUN_B = "run_" + "b" * 24 +SEAL_KEY = bytes(range(32)) +WRONG_SEAL_KEY = bytes(range(1, 33)) + + +def create_live_checkpoint(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.create_live_checkpoint(*args, **kwargs) + + +def write_live_checkpoint(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.write_live_checkpoint(*args, **kwargs) + + +def load_live_checkpoint(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.load_live_checkpoint(*args, **kwargs) + + +def start_active_cell(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.start_active_cell(*args, **kwargs) + + +def update_live_checkpoint(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.update_live_checkpoint(*args, **kwargs) + + +def recover_interrupted_checkpoint(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.recover_interrupted_checkpoint(*args, **kwargs) + + +def finish_active_cell(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.finish_active_cell(*args, **kwargs) def _bindings(**updates: object) -> CheckpointBindings: @@ -62,6 +92,7 @@ def _price_book() -> PriceBook: model_revision="fixture-r1", input_ceiling_usd_per_million_tokens=Decimal("1"), output_ceiling_usd_per_million_tokens=Decimal("1"), + max_output_bytes_per_token=4, ), ), ) @@ -86,6 +117,7 @@ def _pending_call() -> PendingCall: prompt_template_token_allowance=0, provider_framing_token_allowance=2, upstream_output_token_ceilings=(), + upstream_output_bytes_per_token=4, max_output_tokens=5, ) return PendingCall( @@ -403,3 +435,63 @@ def test_corrupt_or_tampered_checkpoint_fails_closed(tmp_path) -> None: path.write_text(json.dumps(payload), encoding="utf-8") with pytest.raises(CheckpointValidationError, match="invalid"): load_live_checkpoint(path, expected_bindings=_bindings()) + + +def test_checkpoint_rejects_publicly_resealed_cost_forgery_and_wrong_key(tmp_path) -> None: + path = tmp_path / "authenticated-checkpoint.json" + receipt = _receipt() + checkpoint = create_live_checkpoint( + bindings=_bindings(), + receipts=(receipt,), + seal_key=SEAL_KEY, + ) + write_live_checkpoint(path, checkpoint, seal_key=SEAL_KEY) + + payload = json.loads(path.read_text(encoding="utf-8")) + payload["committed_cost_usd"] = "0.000003" + payload["receipts"][0]["charged_cost_usd"] = "0.000003" + public_payload = dict(payload) + public_payload.pop("checkpoint_hash") + payload["checkpoint_hash"] = live_module._canonical_hash(public_payload) + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(CheckpointValidationError, match="integrity"): + load_live_checkpoint( + path, + expected_bindings=_bindings(), + seal_key=SEAL_KEY, + ) + + write_live_checkpoint(path, checkpoint, seal_key=SEAL_KEY) + with pytest.raises(CheckpointValidationError, match="integrity"): + load_live_checkpoint( + path, + expected_bindings=_bindings(), + seal_key=WRONG_SEAL_KEY, + ) + + +def test_checkpoint_rejects_legacy_unkeyed_format_and_never_serializes_seal_key( + tmp_path, +) -> None: + path = tmp_path / "checkpoint.json" + checkpoint = create_live_checkpoint(bindings=_bindings(), seal_key=SEAL_KEY) + write_live_checkpoint(path, checkpoint, seal_key=SEAL_KEY) + checkpoint_bytes = path.read_bytes() + + assert SEAL_KEY not in checkpoint_bytes + assert "hmac-sha256:" in checkpoint_bytes.decode("utf-8") + + payload = json.loads(checkpoint_bytes) + payload.pop("checkpoint_format_version") + public_payload = dict(payload) + public_payload.pop("checkpoint_hash") + payload["checkpoint_hash"] = live_module._canonical_hash(public_payload) + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(CheckpointValidationError, match="invalid"): + load_live_checkpoint( + path, + expected_bindings=_bindings(), + seal_key=SEAL_KEY, + ) diff --git a/tests/evals/test_live_estimate.py b/tests/evals/test_live_estimate.py index ddb45b8..f9688d8 100644 --- a/tests/evals/test_live_estimate.py +++ b/tests/evals/test_live_estimate.py @@ -9,8 +9,20 @@ import conclave.evals.live as live_module import conclave.registry as registry_module import conclave.transport as transport_module -from conclave.evals.live import BudgetExceededError, estimate_live_study -from conclave.evals.live_protocols import allocate_stage_caps, stage_call_sequence +from conclave.evals.live import ( + BudgetExceededError, + LiveProviderClient, + _LiveEstimateClient, + estimate_live_study, +) +from conclave.evals.live_protocols import ( + ChatMessage, + StageCall, + allocate_stage_caps, + stage_call_sequence, +) +from conclave.evals.pricing import ModelPrice, PriceBook +from conclave.models import ModelAnswer, TokenUsage from tests.evals.test_live_runner import _live_inputs @@ -152,3 +164,92 @@ async def test_dry_run_rejects_plan_whose_worst_case_exceeds_frozen_ceiling() -> tasks=tasks, price_book=price_book, ) + + +@pytest.mark.asyncio +async def test_estimate_bounds_multibyte_max_expansion_execution_reservation() -> None: + price_book = PriceBook( + snapshot_id="fictional-byte-bound", + captured_at="2026-07-18T12:00:00Z", + currency="USD", + entries=( + ModelPrice( + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + input_ceiling_usd_per_million_tokens=Decimal("1"), + output_ceiling_usd_per_million_tokens=Decimal("1"), + max_output_bytes_per_token=4, + ), + ), + ) + first_call = StageCall( + stage="draft", + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + messages=(ChatMessage(role="user", content="static prompt"),), + max_output_tokens=2, + ) + estimate_client = _LiveEstimateClient(price_book) + estimated_answer = await estimate_client.call(first_call) + estimated_second_call = StageCall( + stage="self_revision", + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + messages=( + ChatMessage( + role="user", + content=f"static wrapper:{estimated_answer.answer}:end", + ), + ), + max_output_tokens=1, + upstream_output_token_ceilings=(2,), + ) + await estimate_client.call(estimated_second_call) + estimated_reservation = estimate_client.reservations[-1] + + pending_reservations = [] + provider_answers = iter(("๐Ÿ˜€๐Ÿ˜€", "done")) + + async def provider(name, model_id, messages, **kwargs): + del messages, kwargs + answer = next(provider_answers) + return ModelAnswer( + name=name, + model_id=model_id, + answer=answer, + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + def checkpoint(pending, receipts) -> None: + del receipts + if pending is not None: + pending_reservations.append(pending.reservation) + + execution_client = LiveProviderClient( + price_book=price_book, + hard_cap_usd=Decimal("1"), + checkpoint=checkpoint, + call_model_func=provider, + ) + actual_answer = await execution_client.call(first_call) + actual_second_call = estimated_second_call.model_copy( + update={ + "messages": ( + ChatMessage( + role="user", + content=f"static wrapper:{actual_answer.answer}:end", + ), + ) + } + ) + await execution_client.call(actual_second_call) + execution_reservation = pending_reservations[-1] + + assert len("๐Ÿ˜€๐Ÿ˜€".encode()) == 2 * price_book.entries[0].max_output_bytes_per_token + assert execution_reservation.input_token_upper_bound <= ( + estimated_reservation.input_token_upper_bound + ) + assert execution_reservation.reserved_cost_usd <= estimated_reservation.reserved_cost_usd diff --git a/tests/evals/test_live_gateway.py b/tests/evals/test_live_gateway.py index 26c25fa..99a8658 100644 --- a/tests/evals/test_live_gateway.py +++ b/tests/evals/test_live_gateway.py @@ -36,6 +36,7 @@ def _price_book( model_revision="fixture-r1", input_ceiling_usd_per_million_tokens=Decimal(input_rate), output_ceiling_usd_per_million_tokens=Decimal(output_rate), + max_output_bytes_per_token=4, ), ), ) diff --git a/tests/evals/test_live_replay.py b/tests/evals/test_live_replay.py index 0a70c12..bbb23bb 100644 --- a/tests/evals/test_live_replay.py +++ b/tests/evals/test_live_replay.py @@ -33,6 +33,7 @@ FIXTURE_DIR = Path(__file__).parents[1] / "fixtures/evals/live_smoke" FAKE_KEY_ENV = "CONCLAVE_FAKE_TEST_KEY" FAKE_KEY = "fixture-only-opaque-credential" +CHECKPOINT_SEAL_KEY = bytes(range(32)) def _canonical_bytes(value: object) -> bytes: @@ -143,6 +144,7 @@ async def configured_call_model(name, model_id, messages, **kwargs): tasks=tasks, price_book=price_book, checkpoint_path=checkpoint_path, + checkpoint_seal_key=CHECKPOINT_SEAL_KEY, call_model_func=configured_call_model, ) checkpoint = load_live_checkpoint( @@ -152,6 +154,7 @@ async def configured_call_model(name, model_id, messages, **kwargs): price_book, hard_cap_usd=LIVE_HARD_CAP_USD, ), + seal_key=CHECKPOINT_SEAL_KEY, ) return study_run, checkpoint diff --git a/tests/evals/test_live_runner.py b/tests/evals/test_live_runner.py index 9fd25ef..11a5c9a 100644 --- a/tests/evals/test_live_runner.py +++ b/tests/evals/test_live_runner.py @@ -9,13 +9,11 @@ import pytest import conclave.evals.dataset as dataset_module +import conclave.evals.live as live_module import conclave.evals.runner as runner_module from conclave.evals.live import ( CheckpointValidationError, build_checkpoint_bindings, - create_live_checkpoint, - load_live_checkpoint, - write_live_checkpoint, ) from conclave.evals.live_protocols import LIVE_PROTOCOL_REGISTRY, stage_call_sequence from conclave.evals.models import ( @@ -32,12 +30,33 @@ ) from conclave.evals.pricing import ModelPrice, PriceBook, hash_price_entries from conclave.evals.protocols import CONDITION_IDS, build_study_manifest -from conclave.evals.runner import RunValidationError, run_live_study, validate_run_records +from conclave.evals.runner import RunValidationError, validate_run_records from conclave.models import ModelAnswer, TokenUsage DIGEST = "sha256:" + "a" * 64 HARD_CAP = Decimal("10.00") LIVE_FIXTURE_CELL_CEILING = 6144 +SEAL_KEY = bytes(range(32)) + + +def create_live_checkpoint(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.create_live_checkpoint(*args, **kwargs) + + +def write_live_checkpoint(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.write_live_checkpoint(*args, **kwargs) + + +def load_live_checkpoint(*args, **kwargs): + kwargs.setdefault("seal_key", SEAL_KEY) + return live_module.load_live_checkpoint(*args, **kwargs) + + +async def run_live_study(*args, **kwargs): + kwargs.setdefault("checkpoint_seal_key", SEAL_KEY) + return await runner_module.run_live_study(*args, **kwargs) def _valid_verdict_extraction_text() -> str: @@ -109,6 +128,7 @@ def _live_inputs( model_revision=member.model_revision, input_ceiling_usd_per_million_tokens=Decimal(rate), output_ceiling_usd_per_million_tokens=Decimal(rate), + max_output_bytes_per_token=4, ) for roster in rosters for member in roster.members diff --git a/tests/evals/test_pricing.py b/tests/evals/test_pricing.py index 9916408..fc04b6c 100644 --- a/tests/evals/test_pricing.py +++ b/tests/evals/test_pricing.py @@ -30,7 +30,7 @@ DIGEST = "sha256:" + "a" * 64 PRICE_FIXTURE = Path(__file__).parents[1] / "fixtures/evals/live_smoke/price_book.json" -EXPECTED_PRICE_HASH = "sha256:df85e17c12985ada8a350c6f2adc1778a0218bf3dc70cf3204ac8d1057939aac" +EXPECTED_PRICE_HASH = "sha256:46a29d0180e897fd8ba315781b9180121a4509226b8dfb8084164515e0efa53f" def _payload() -> dict[str, object]: @@ -182,6 +182,36 @@ def test_price_book_requires_usd_positive_pessimistic_rates() -> None: _book(payload) +def test_price_book_requires_positive_external_output_byte_bound() -> None: + missing = _payload() + entries = missing["entries"] + assert isinstance(entries, list) + entries[0].pop("max_output_bytes_per_token", None) + with pytest.raises(ValidationError, match="max_output_bytes_per_token"): + _book(missing) + + for invalid in (0, -1, True): + payload = _payload() + entries = payload["entries"] + assert isinstance(entries, list) + entries[0]["max_output_bytes_per_token"] = invalid + with pytest.raises(ValidationError, match="max_output_bytes_per_token"): + _book(payload) + + +def test_output_byte_bound_is_snapshot_hashed_and_drift_rejected() -> None: + book = _book() + price_hash = hash_price_entries(book.entries) + changed = book.entries[0].model_copy( + update={"max_output_bytes_per_token": book.entries[0].max_output_bytes_per_token + 1} + ) + drifted = book.model_copy(update={"entries": (changed, *book.entries[1:])}) + + assert hash_price_entries(drifted.entries) != price_hash + with pytest.raises(ValueError, match="prices_hash"): + validate_price_book(drifted, frozen_design=_design(prices_hash=price_hash)) + + def test_call_reservation_rounds_up_and_covers_input_output_and_framing() -> None: price = _book().entries[0] @@ -191,15 +221,16 @@ def test_call_reservation_rounds_up_and_covers_input_output_and_framing() -> Non prompt_template_token_allowance=17, provider_framing_token_allowance=11, upstream_output_token_ceilings=(50, 60), + upstream_output_bytes_per_token=price.max_output_bytes_per_token, max_output_tokens=75, ) assert isinstance(reservation, CallReservation) - assert reservation.input_token_upper_bound == 239 + assert reservation.input_token_upper_bound == 569 assert reservation.output_token_upper_bound == 75 - assert reservation.input_cost_upper_bound_usd == Decimal("0.000295061513") + assert reservation.input_cost_upper_bound_usd == Decimal("0.000702468623") assert reservation.output_cost_upper_bound_usd == Decimal("0.000342591825") - assert reservation.reserved_cost_usd == Decimal("0.000638") + assert reservation.reserved_cost_usd == Decimal("0.001046") assert all( type(value) is Decimal for value in ( diff --git a/tests/fixtures/evals/live_smoke/manifest.json b/tests/fixtures/evals/live_smoke/manifest.json index 1c77342..0c6dcc8 100644 --- a/tests/fixtures/evals/live_smoke/manifest.json +++ b/tests/fixtures/evals/live_smoke/manifest.json @@ -140,17 +140,17 @@ "snapshot_id": "fictional-live-smoke-prices-2026-07-18", "captured_at": "2026-07-18T12:00:00Z", "currency": "USD", - "prices_hash": "sha256:df85e17c12985ada8a350c6f2adc1778a0218bf3dc70cf3204ac8d1057939aac" + "prices_hash": "sha256:46a29d0180e897fd8ba315781b9180121a4509226b8dfb8084164515e0efa53f" }, "approved_spend_ceiling_usd": 10.0, "preregistration_id": null, "preregistration_hash": null }, - "frozen_design_hash": "sha256:1870bb73e7e57f7b762cbd76898ab07267c1948a799f01b86a5e05a9be9581af", + "frozen_design_hash": "sha256:bb117363acd9a015f0d13f7430c3749126911537dbd483f9b563eb2c74a54514", "planned_runs": [ { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_1514260bbb609559e7fac798", + "planned_run_id": "run_38f8607545e9168ca253d45c", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -160,7 +160,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_250a6a43f71c819882567f2f", + "planned_run_id": "run_57d6a540ef061ffdba68040e", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -170,7 +170,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_03e305fc346582f50bd98a86", + "planned_run_id": "run_b01476152312843f5584d5a7", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -180,7 +180,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_5ab2ccbb3c51c7c9b798847b", + "planned_run_id": "run_b2335407d23f08fc4fb07ac3", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -190,7 +190,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_56ace8153142d9801dc5d6cb", + "planned_run_id": "run_96661a0f080d7665dfaee689", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -200,7 +200,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_9663dc1ddd27fe7f45e59c47", + "planned_run_id": "run_c12b04e765d5ed8bc863d4bf", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -210,7 +210,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_c75cdbf92f852ab704e4e831", + "planned_run_id": "run_48f7c108af408348a87dba66", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -220,7 +220,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_0dad25939ce03c3ca19f7dcf", + "planned_run_id": "run_8afec0c9ff7253fa2e55a342", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -230,7 +230,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_a6934c752d97ad8fced5c4b4", + "planned_run_id": "run_fa3c0d038e4c0d0b674caf6d", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -240,7 +240,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_1e19e08e1ce475e8d6fd9e64", + "planned_run_id": "run_e0a25669436b90c26bd402a2", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -250,7 +250,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_fa0d8ba45a49c759dd057670", + "planned_run_id": "run_1dec38cf81eb92d80e24c53e", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -260,7 +260,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_f789cd855c22413292bf37be", + "planned_run_id": "run_78794c4ef1c3cb4c4e9e2f91", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", diff --git a/tests/fixtures/evals/live_smoke/price_book.json b/tests/fixtures/evals/live_smoke/price_book.json index a9bedb5..b0c1665 100644 --- a/tests/fixtures/evals/live_smoke/price_book.json +++ b/tests/fixtures/evals/live_smoke/price_book.json @@ -10,7 +10,8 @@ "model_id": "fictional-model-a", "model_revision": "fixture-r1", "input_ceiling_usd_per_million_tokens": "1.234567", - "output_ceiling_usd_per_million_tokens": "4.567891" + "output_ceiling_usd_per_million_tokens": "4.567891", + "max_output_bytes_per_token": 4 }, { "schema_version": "conclave_eval_v1", @@ -18,7 +19,8 @@ "model_id": "fictional-model-b", "model_revision": "fixture-r2", "input_ceiling_usd_per_million_tokens": "2.000001", - "output_ceiling_usd_per_million_tokens": "6.000001" + "output_ceiling_usd_per_million_tokens": "6.000001", + "max_output_bytes_per_token": 4 }, { "schema_version": "conclave_eval_v1", @@ -26,7 +28,8 @@ "model_id": "fictional-model-c", "model_revision": "fixture-r3", "input_ceiling_usd_per_million_tokens": "3.000001", - "output_ceiling_usd_per_million_tokens": "9.000001" + "output_ceiling_usd_per_million_tokens": "9.000001", + "max_output_bytes_per_token": 4 } ] } diff --git a/tests/fixtures/evals/live_smoke/replay.json b/tests/fixtures/evals/live_smoke/replay.json index 434c58e..882aae9 100644 --- a/tests/fixtures/evals/live_smoke/replay.json +++ b/tests/fixtures/evals/live_smoke/replay.json @@ -1,6 +1,6 @@ { "schema_version": "conclave_replay_v1", - "base_manifest_hash": "sha256:72472cd0faa0c9de49a195ae70a797a4c763e6e20c96ad982a89bb4bfc55a076", + "base_manifest_hash": "sha256:46d0641b5659c9feee4802a9cefbd155e1f83513791352b75c747f7ebe27ef64", "records": [ { "request_hash": "sha256:cc57d8d8ed8963f3537727956243f353427936b20d5365ebb8438e91a93340e1", From 6deccf6628bb15f18229c62b7f67521a7172664e Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 23:01:00 -0400 Subject: [PATCH 14/14] fix(evals): close live accounting gaps --- ...-07-18-h1-live-evaluation-runner-design.md | 21 ++++---- src/conclave/evals/live.py | 53 +++++++++++++++++-- src/conclave/evals/runner.py | 13 ++++- src/conclave/verdict.py | 2 +- src/conclave/verdict_synthesis.py | 20 +++++-- tests/evals/test_live_estimate.py | 34 ++++++++++++ tests/evals/test_live_gateway.py | 47 +++++++++++++++- tests/evals/test_live_runner.py | 11 ++++ tests/fixtures/evals/live_smoke/manifest.json | 28 +++++----- tests/fixtures/evals/live_smoke/replay.json | 2 +- tests/test_verdict_synthesis.py | 24 +++++++++ 11 files changed, 220 insertions(+), 35 deletions(-) diff --git a/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md b/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md index 5b6e3f0..4b1d15a 100644 --- a/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md +++ b/docs/plans/2026-07-18-h1-live-evaluation-runner-design.md @@ -50,9 +50,11 @@ the snapshot ID, capture time, and currency must also match. Duplicate or missin unknown models, non-USD currency, and nonpositive rates fail before key resolution or network access. -Execution reservations use the exact UTF-8 byte length of resolved messages plus provider -framing and the call's output-token ceiling; treating every input byte as a possible token is -pessimistic. Dry-run cannot know future upstream text, so it removes its internal sentinels and +Execution reservations use the exact UTF-8 byte length of resolved messages and any native +structured-output contract plus provider framing and the call's output-token ceiling; treating +every input byte as a possible token is pessimistic. Verdict validation errors are capped by +UTF-8 bytes before the one repair prompt, and dry-run reserves that full cap. Dry-run cannot know +future upstream text, so it removes its internal sentinels and reserves each upstream token ceiling multiplied by the maximum `max_output_bytes_per_token` attestation in the frozen price book. That attestation is part of the canonical price hash, so drift invalidates the manifest binding. The price book uses @@ -67,12 +69,13 @@ Before network I/O: 4. Atomically persist a pending call with the reservation. 5. Invoke `call_model` with that stage's `max_output_tokens`. -Afterward, complete usage is priced from provider token receipts. The runner commits the -actual calculated cost when it is within the reservation; missing usage commits the full -reservation. A usage count or calculated cost above the reservation is a fail-closed -`reservation_breach`: the full reservation is charged, the cell is non-successful, and no -new call is scheduled. The final artifacts distinguish usage-priced cost from pessimistically -charged cost. +Afterward, complete usage is priced from provider token receipts. Any provider-reported total +above prompt plus completion is charged pessimistically at the higher frozen rate; a total below +those components is a reconciliation failure. Missing usage commits the full reservation. A +usage count or calculated cost above the reservation is a fail-closed `reservation_breach`: the +actual usage-priced overage is preserved as evidence, the cell is non-successful, and no new call +is scheduled. The final artifacts distinguish usage-priced cost from pessimistically charged +cost. ## Six conditions diff --git a/src/conclave/evals/live.py b/src/conclave/evals/live.py index da195a7..ca9d8fd 100644 --- a/src/conclave/evals/live.py +++ b/src/conclave/evals/live.py @@ -20,6 +20,7 @@ from conclave.models import ModelAnswer, TokenUsage from conclave.providers import _receipt_error_category, call_model from conclave.registry import PROVIDER_ENV_VARS +from conclave.verdict_synthesis import VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES from .live_protocols import LIVE_PROTOCOL_REGISTRY, StageCall, execute_live_condition from .models import ( @@ -102,6 +103,8 @@ class ProviderCallCostBasis(EvalModel): source: CostBasisSource prompt_tokens: int | None = Field(default=None, ge=0) completion_tokens: int | None = Field(default=None, ge=0) + total_tokens: int | None = Field(default=None, ge=0) + unattributed_tokens: int | None = Field(default=None, ge=0) input_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) output_ceiling_usd_per_million_tokens: Decimal = Field(gt=0) @@ -593,15 +596,27 @@ def finish_active_cell( def _prompt_token_upper_bound(call: StageCall) -> int: - """Bound prompt content by UTF-8 bytes, conservatively above BPE tokens.""" + """Bound messages and native structured-output content by UTF-8 bytes.""" - return sum(len(message.content.encode("utf-8")) for message in call.messages) + message_bytes = sum(len(message.content.encode("utf-8")) for message in call.messages) + if call.output_contract is None: + return message_bytes + contract_bytes = len( + json.dumps( + call.output_contract.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ) + return message_bytes + contract_bytes def _provider_framing_token_allowance(call: StageCall) -> int: """Reserve fixed per-request and per-message provider framing overhead.""" - return 64 + (16 * len(call.messages)) + contract_wrapper_allowance = 256 if call.output_contract is not None else 0 + return 64 + (16 * len(call.messages)) + contract_wrapper_allowance def _resolve_call_price(price_book: PriceBook, call: StageCall) -> ModelPrice: @@ -637,7 +652,15 @@ def _reserve_stage_call( ) +def _unattributed_usage_tokens(usage: TokenUsage) -> int: + attributed = usage.prompt_tokens + usage.completion_tokens + if usage.total_tokens < attributed: + raise ValueError("provider total_tokens is smaller than its attributed usage") + return usage.total_tokens - attributed + + def _reported_usage_cost(price: ModelPrice, usage: TokenUsage) -> Decimal: + unattributed = _unattributed_usage_tokens(usage) precision = max( 64, len(str(usage.prompt_tokens)) @@ -646,12 +669,23 @@ def _reported_usage_cost(price: ModelPrice, usage: TokenUsage) -> Decimal: len(str(usage.completion_tokens)) + len(price.output_ceiling_usd_per_million_tokens.as_tuple().digits) + 20, + len(str(usage.total_tokens)) + + max( + len(price.input_ceiling_usd_per_million_tokens.as_tuple().digits), + len(price.output_ceiling_usd_per_million_tokens.as_tuple().digits), + ) + + 20, ) with localcontext() as context: context.prec = precision cost = ( Decimal(usage.prompt_tokens) * price.input_ceiling_usd_per_million_tokens + Decimal(usage.completion_tokens) * price.output_ceiling_usd_per_million_tokens + + Decimal(unattributed) + * max( + price.input_ceiling_usd_per_million_tokens, + price.output_ceiling_usd_per_million_tokens, + ) ) / TOKENS_PER_MILLION return cost.quantize(USD_MICROCENT, rounding=ROUND_CEILING) @@ -742,9 +776,12 @@ def _reserve(self, call: StageCall, price: ModelPrice) -> CallReservation: @staticmethod def _usage_breaches(usage: TokenUsage, reservation: CallReservation) -> bool: + unattributed = _unattributed_usage_tokens(usage) return ( usage.prompt_tokens > reservation.input_token_upper_bound - or usage.completion_tokens > reservation.output_token_upper_bound + or usage.completion_tokens + unattributed > reservation.output_token_upper_bound + or usage.total_tokens + > reservation.input_token_upper_bound + reservation.output_token_upper_bound ) async def call(self, call: StageCall) -> ModelAnswer: @@ -816,6 +853,8 @@ async def call(self, call: StageCall) -> ModelAnswer: source="reported_usage", prompt_tokens=usage.prompt_tokens, completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + unattributed_tokens=_unattributed_usage_tokens(usage), input_ceiling_usd_per_million_tokens=( reservation.input_ceiling_usd_per_million_tokens ), @@ -914,6 +953,12 @@ async def call(self, call: StageCall) -> ModelAnswer: ) if call.stage == "verdict_repair": raise _EstimateMaxGraphComplete + if call.stage == "verdict": + return ModelAnswer( + name=call.provider_id, + model_id=call.model_id, + error="x" * VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES, + ) placeholder = f"__conclave_estimated_output_{len(self._upstream_placeholders):08d}__" self._upstream_placeholders[placeholder] = call.max_output_tokens return ModelAnswer( diff --git a/src/conclave/evals/runner.py b/src/conclave/evals/runner.py index 35a6859..3f1bc49 100644 --- a/src/conclave/evals/runner.py +++ b/src/conclave/evals/runner.py @@ -4,6 +4,7 @@ import asyncio import os +import stat import time from collections import Counter from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence @@ -70,7 +71,7 @@ class RunValidationError(ValueError): def _checkpoint_lifecycle_lease(checkpoint_path: Path) -> Iterator[None]: """Hold one nonblocking process lease for the checkpoint's full lifecycle.""" - if _fcntl is None: + if _fcntl is None or not hasattr(os, "O_NOFOLLOW"): raise CheckpointValidationError( "checkpoint filesystem leases are unsupported on this platform" ) @@ -79,10 +80,18 @@ def _checkpoint_lifecycle_lease(checkpoint_path: Path) -> Iterator[None]: descriptor: int | None = None acquired = False try: - flags = os.O_CREAT | os.O_RDWR + flags = os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW if hasattr(os, "O_CLOEXEC"): flags |= os.O_CLOEXEC descriptor = os.open(lock_path, flags, 0o600) + lock_stat = os.fstat(descriptor) + if ( + not stat.S_ISREG(lock_stat.st_mode) + or lock_stat.st_nlink != 1 + or lock_stat.st_uid != os.getuid() + or stat.S_IMODE(lock_stat.st_mode) & (stat.S_IRWXG | stat.S_IRWXO) + ): + raise OSError("unsafe checkpoint lease file") _fcntl.flock(descriptor, _fcntl.LOCK_EX | _fcntl.LOCK_NB) acquired = True except (AttributeError, OSError): diff --git a/src/conclave/verdict.py b/src/conclave/verdict.py index 8c62a1f..d1b7767 100644 --- a/src/conclave/verdict.py +++ b/src/conclave/verdict.py @@ -57,7 +57,7 @@ # audit can tell WHICH extractor wording produced a given clustering. Opaque # string; only equality/inequality is meaningful. Bump on any change to the # extraction system prompt in :mod:`conclave.verdict_synthesis`. -VERDICT_EXTRACTION_PROMPT_VERSION = "2" +VERDICT_EXTRACTION_PROMPT_VERSION = "3" class CouncilPosition(BaseModel): diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index 362b4da..c2598c3 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -84,6 +84,7 @@ __all__ = [ "VERDICT_EXTRACTION_PROMPT_VERSION", + "VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES", "VerdictSynthesisResult", "extract_verdict", "verdict_extraction_json_schema", @@ -91,12 +92,25 @@ logger = get_logger("verdict_synthesis") +# Repair prompts are paid input. Bound validation detail by UTF-8 bytes so a +# malformed structured response cannot amplify into an unbounded second call. +VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES = 2048 + # Recorded reasons for an absent verdict (DD-2 verdict-absent rule). Named as # constants so the three call sites and the test suite share the exact strings. _REASON_TOO_FEW = "fewer than 2 responding members" _REASON_OPEN_ENDED = "open-ended prompt (no decision/review to adjudicate)" _REASON_EXTRACTION_FAILED = "verdict extraction failed schema validation" + +def _bounded_repair_error(detail: object) -> str: + scrubbed = redact(str(detail)) + encoded = scrubbed.encode("utf-8") + if len(encoded) <= VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES: + return scrubbed + return encoded[:VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES].decode("utf-8", errors="ignore") + + # The extraction system prompt. Versioned via VERDICT_EXTRACTION_PROMPT_VERSION # (bump that constant in verdict.py on any wording change here). It instructs the # model to emit ONLY the structured judgment per the schema and โ€” load-bearing for @@ -310,18 +324,18 @@ def _parse_and_validate(answer: ModelAnswer | None) -> tuple[VerdictExtractionMo """ if answer is None or answer.error or not answer.answer or not answer.answer.strip(): detail = answer.error if (answer and answer.error) else "empty extractor response" - return None, redact(str(detail)) + return None, _bounded_repair_error(detail) candidate = _strip_code_fence(answer.answer) try: data = json.loads(candidate) except json.JSONDecodeError as exc: - return None, redact(f"response was not valid JSON: {exc}") + return None, _bounded_repair_error(f"response was not valid JSON: {exc}") try: return VerdictExtractionModel.model_validate(data), "" except ValidationError as exc: - return None, redact(f"JSON did not match the verdict schema: {exc}") + return None, _bounded_repair_error(f"JSON did not match the verdict schema: {exc}") def _member_vote_sequence( diff --git a/tests/evals/test_live_estimate.py b/tests/evals/test_live_estimate.py index f9688d8..170a137 100644 --- a/tests/evals/test_live_estimate.py +++ b/tests/evals/test_live_estimate.py @@ -23,6 +23,7 @@ ) from conclave.evals.pricing import ModelPrice, PriceBook from conclave.models import ModelAnswer, TokenUsage +from conclave.verdict_synthesis import VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES from tests.evals.test_live_runner import _live_inputs @@ -128,6 +129,39 @@ async def test_dry_run_prices_the_optional_elite_verdict_repair_max_graph() -> N assert estimate.maximum_call_count == expected_calls +@pytest.mark.asyncio +async def test_dry_run_models_maximum_bounded_verdict_repair_error() -> None: + price_book = PriceBook( + snapshot_id="fictional-repair-bound", + captured_at="2026-07-18T12:00:00Z", + currency="USD", + entries=( + ModelPrice( + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + input_ceiling_usd_per_million_tokens=Decimal("1"), + output_ceiling_usd_per_million_tokens=Decimal("1"), + max_output_bytes_per_token=4, + ), + ), + ) + client = _LiveEstimateClient(price_book) + verdict = StageCall( + stage="verdict", + provider_id="fictional-provider", + model_id="fictional/model", + model_revision="fixture-r1", + messages=(ChatMessage(role="user", content="extract verdict"),), + max_output_tokens=2, + ) + + answer = await client.call(verdict) + + assert answer.error is not None + assert len(answer.error.encode("utf-8")) == VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES + + @pytest.mark.asyncio async def test_dry_run_breaks_down_upper_bound_by_roster_and_condition() -> None: tasks, manifest, price_book = _live_inputs() diff --git a/tests/evals/test_live_gateway.py b/tests/evals/test_live_gateway.py index 99a8658..7ccd645 100644 --- a/tests/evals/test_live_gateway.py +++ b/tests/evals/test_live_gateway.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json from decimal import Decimal import pytest @@ -105,6 +106,7 @@ async def test_gateway_passes_stage_output_contract_to_provider() -> None: strict=True, ) observed: list[OutputContract | None] = [] + reservations = [] async def fake_call_model(name, model_id, messages, **kwargs): del messages @@ -120,13 +122,26 @@ async def fake_call_model(name, model_id, messages, **kwargs): price_book=_price_book(input_rate="1", output_rate="1"), hard_cap_usd=Decimal("1"), call_model_func=fake_call_model, - checkpoint=lambda pending, receipts: None, + checkpoint=lambda pending, receipts: ( + reservations.append(pending.reservation) if pending is not None else None + ), ) call = _stage_call(output_contract=contract) await client.call(call) assert observed == [contract] + serialized_contract_bytes = len( + json.dumps( + contract.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ) + assert reservations[0].prompt_token_upper_bound >= ( + len(call.messages[0].content.encode("utf-8")) + serialized_contract_bytes + ) @pytest.mark.asyncio @@ -295,6 +310,36 @@ async def fake_call_model(name, model_id, messages, **kwargs): assert client.committed_cost_usd == receipt.charged_cost_usd +@pytest.mark.asyncio +async def test_gateway_accounts_for_provider_reported_unattributed_total_tokens() -> None: + async def fake_call_model(name, model_id, messages, **kwargs): + del messages, kwargs + return ModelAnswer( + name=name, + model_id=model_id, + answer="provider reported hidden billed tokens", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=10_000), + ) + + client = LiveProviderClient( + price_book=_price_book(input_rate="1", output_rate="2"), + hard_cap_usd=Decimal("1"), + call_model_func=fake_call_model, + checkpoint=lambda pending, receipts: None, + ) + + with pytest.raises(ReservationBreachError, match="reservation"): + await client.call(_stage_call(cap=5)) + + receipt = client.receipts[-1] + assert receipt.cost_basis.total_tokens == 10_000 + assert receipt.cost_basis.unattributed_tokens == 9_998 + assert receipt.charged_cost_usd == Decimal("0.019999") + assert receipt.charged_cost_usd > receipt.reserved_cost_usd + assert receipt.error_category == "reservation_breach" + assert client.stopped is True + + def test_token_usage_rejects_negative_counts() -> None: with pytest.raises(ValidationError): TokenUsage(prompt_tokens=-1, completion_tokens=0, total_tokens=0) diff --git a/tests/evals/test_live_runner.py b/tests/evals/test_live_runner.py index 11a5c9a..7dcc11a 100644 --- a/tests/evals/test_live_runner.py +++ b/tests/evals/test_live_runner.py @@ -420,6 +420,17 @@ def test_live_runner_allows_only_one_process_per_checkpoint(tmp_path) -> None: assert all(process.exitcode == 0 for process in processes) +def test_live_runner_rejects_symlinked_lifecycle_lock(tmp_path) -> None: + checkpoint_path = tmp_path / "symlinked-checkpoint.json" + checkpoint_path.write_text("{}", encoding="utf-8") + lock_path = checkpoint_path.with_name(f"{checkpoint_path.name}.lock") + lock_path.symlink_to(checkpoint_path) + + with pytest.raises(CheckpointValidationError, match="lease"): + with runner_module._checkpoint_lifecycle_lease(checkpoint_path): + pass + + @pytest.mark.asyncio async def test_live_runner_fails_closed_when_filesystem_leases_are_unsupported( tmp_path, monkeypatch diff --git a/tests/fixtures/evals/live_smoke/manifest.json b/tests/fixtures/evals/live_smoke/manifest.json index 0c6dcc8..6596ba1 100644 --- a/tests/fixtures/evals/live_smoke/manifest.json +++ b/tests/fixtures/evals/live_smoke/manifest.json @@ -72,7 +72,7 @@ "independent_synthesis": "live_synthesis_2026-06-29", "critique_only": "live_critique_1_synthesis_2026-06-29", "revision_only": "live_revision_v1_synthesis_2026-06-29", - "elite_full": "elite_1_synthesis_2026-06-29_verdict_2" + "elite_full": "elite_1_synthesis_2026-06-29_verdict_3" }, "condition_protocol_versions": { "single_frontier": "single_frontier_live_v2", @@ -146,11 +146,11 @@ "preregistration_id": null, "preregistration_hash": null }, - "frozen_design_hash": "sha256:bb117363acd9a015f0d13f7430c3749126911537dbd483f9b563eb2c74a54514", + "frozen_design_hash": "sha256:a811130bd4a986fdacf090e66192818fa235819bc2e4f9401714e42bcb41bdac", "planned_runs": [ { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_38f8607545e9168ca253d45c", + "planned_run_id": "run_c1a93e940605e8292b67094e", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -160,7 +160,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_57d6a540ef061ffdba68040e", + "planned_run_id": "run_ed6a702b1b84a176d5f72a93", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -170,7 +170,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_b01476152312843f5584d5a7", + "planned_run_id": "run_f5fe0ed0dd5f1dae988451dd", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -180,7 +180,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_b2335407d23f08fc4fb07ac3", + "planned_run_id": "run_bc95d6a828b012df89b302b5", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -190,7 +190,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_96661a0f080d7665dfaee689", + "planned_run_id": "run_b9d0b8fe2b475b93dbf25a2c", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -200,7 +200,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_c12b04e765d5ed8bc863d4bf", + "planned_run_id": "run_03a8814ed89c23591f147c7a", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -210,7 +210,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_48f7c108af408348a87dba66", + "planned_run_id": "run_5658e9381ae1544529e0e864", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -220,7 +220,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_8afec0c9ff7253fa2e55a342", + "planned_run_id": "run_0bdce4dc58acd7a2e6f6a18b", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -230,7 +230,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_fa3c0d038e4c0d0b674caf6d", + "planned_run_id": "run_86382226fc3cc0317ff6cbe3", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -240,7 +240,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_e0a25669436b90c26bd402a2", + "planned_run_id": "run_747e55eb644e03a5f1d8c156", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -250,7 +250,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_1dec38cf81eb92d80e24c53e", + "planned_run_id": "run_51bcfb311c31532f81ac0e5a", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -260,7 +260,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_78794c4ef1c3cb4c4e9e2f91", + "planned_run_id": "run_433504af776bc9a6d70694d3", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", diff --git a/tests/fixtures/evals/live_smoke/replay.json b/tests/fixtures/evals/live_smoke/replay.json index 882aae9..e3f6bf7 100644 --- a/tests/fixtures/evals/live_smoke/replay.json +++ b/tests/fixtures/evals/live_smoke/replay.json @@ -1,6 +1,6 @@ { "schema_version": "conclave_replay_v1", - "base_manifest_hash": "sha256:46d0641b5659c9feee4802a9cefbd155e1f83513791352b75c747f7ebe27ef64", + "base_manifest_hash": "sha256:50a3e7cb08e8df6da6cdd95fba6404dcb7cefcfcf0cfd5188ec45285df7b1672", "records": [ { "request_hash": "sha256:cc57d8d8ed8963f3537727956243f353427936b20d5365ebb8438e91a93340e1", diff --git a/tests/test_verdict_synthesis.py b/tests/test_verdict_synthesis.py index 6ebbb46..b2fbe72 100644 --- a/tests/test_verdict_synthesis.py +++ b/tests/test_verdict_synthesis.py @@ -35,6 +35,7 @@ from conclave.verdict import CouncilVerdict from conclave.verdict_synthesis import ( VERDICT_EXTRACTION_PROMPT_VERSION, + VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES, VerdictSynthesisResult, extract_verdict, verdict_extraction_json_schema, @@ -497,6 +498,29 @@ def test_repair_message_includes_validation_errors(monkeypatch): assert "valid json" in repair_text or "error" in repair_text +def test_repair_message_caps_schema_errors_by_utf8_bytes(monkeypatch): + malformed = json.dumps({"positions": ["\U0001f600" * 2048]}) + fake = _ScriptedExtractor(malformed, json.dumps(_payload())) + _install(monkeypatch, fake) + + result = __import__("asyncio").run( + extract_verdict( + "Choose.", + [member("alpha", "A", answer_id="alpha-1"), member("beta", "B", answer_id="beta-1")], + synthesizer_name=SYNTH_NAME, + synthesizer_model_id=SYNTH_MODEL_ID, + ) + ) + + assert result.verdict is not None + repair_error = ( + fake.calls[1]["messages"][-1]["content"] + .split("The problem was:\n", 1)[1] + .split("\n\nReturn only", 1)[0] + ) + assert len(repair_error.encode("utf-8")) <= VERDICT_REPAIR_ERROR_DETAIL_MAX_BYTES + + def test_repair_once_then_success(monkeypatch): """First payload invalid, second valid โ†’ verdict assembled after one repair.""" members = [