From 94e3f2a55c8ab61d6e9fb23ebfbc5b9e6f04d2dc Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 17:50:43 -0400 Subject: [PATCH 01/13] docs(evals): plan H1 method hardening and pilot --- .../2026-07-17-h1-method-hardening-design.md | 41 ++++++++++++++++++ docs/plans/2026-07-17-h1-method-hardening.md | 42 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 docs/plans/2026-07-17-h1-method-hardening-design.md create mode 100644 docs/plans/2026-07-17-h1-method-hardening.md diff --git a/docs/plans/2026-07-17-h1-method-hardening-design.md b/docs/plans/2026-07-17-h1-method-hardening-design.md new file mode 100644 index 0000000..25e0306 --- /dev/null +++ b/docs/plans/2026-07-17-h1-method-hardening-design.md @@ -0,0 +1,41 @@ +# H1 Method Hardening Design + +**Linear:** DSE-708 +**Dependency:** PR #52 at `4365c28` +**Gate:** Required before any paid pilot call + +## Decision + +Keep the existing H1 substrate as the execution primitive, then add a frozen study-method contract and a balanced 24-task exploratory pack. The pilot estimates variance, grader reliability, leakage, latency, and cost; it cannot produce a product GO or KILL. + +## Frozen study contract + +Every manifest must bind the exact base commit, study phase, task-family map, two roster specifications, provider/model revisions, condition prompt and protocol versions, generation settings, public-task and private-key hashes, rubric and grader-instruction hashes, evaluator and analysis-code versions, randomization and bootstrap settings, timeout/retry policy, exclusion/deviation policy, price snapshot, approved spend ceiling, and preregistration identifier/hash. + +Confirmatory reports fail closed if any frozen field is missing or its observed receipt differs. Exploratory manifests are explicit and cannot be promoted after results exist. + +## Experimental unit and randomization + +The unit of analysis is the task. Roster and replicate are repeated measurements averaged within task. Create every task x roster x condition x replicate cell before execution. Derive an independent condition permutation from the master seed plus task and roster IDs; interleave blocks so time or provider order cannot align with one condition. All failed, timed-out, malformed, over-budget, abstained, incomplete, ungraded, and unresolved cells remain failures. + +## Grading contract + +Two independent graders score every successful output. Atomic records include rubric item, error category and severity, severe-error flag, holistic dimensions, reviewer seconds, confidence or abstention, rubric version/hash, grader batch/order, and condition/provider guess. Adjudication sees only genuine disagreements, cites every source record, and never overwrites raw judgments. + +Non-success records bypass the human queue and are automatic failures. Successful grader views strip provider, model, condition, mode, and formatting fingerprints; normalize presentation deterministically; run a leakage scan; and retain a separate hashed/access-controlled blind map. + +## Analysis and gates + +Pilot reporting uses failure-inclusive Wilson rates, task-clustered paired bootstrap differences with roster/replicate averaging, raw agreement, prevalence, kappa, adjudication rate, severe errors, reviewer effort, token/cost distributions, latency distributions, deviations, and leakage guesses. + +Reliability is undefined—not perfect—when expected agreement is one. The pilot is method-ready only with at least 95% double grading, raw agreement at least 80%, kappa at least 0.60 overall and no family below 0.50, adjudication at most 20%, and no material leakage. Otherwise redesign. + +Confirmation freezes one strongest baseline. GO requires a point gain of at least 10 percentage points with the paired 95% interval above zero, severe-error noninferiority within +2 points, readiness noninferiority within +5 points, reviewer-effort and latency gates, positive direction in every family and roster, and failure-inclusive validity. Simpler-baseline equivalence, harm, weak reliability, or post-hoc changes trigger redesign or kill per DSE-708. + +## Pilot pack + +Use 24 synthetic, current-fact-free tasks: 12 operational-execution and 12 organizational-stewardship, balanced across six subfamilies, three difficulty tiers, and `ready`/`not_ready`/`indeterminate`. Public packets and grader-only keys are separate. Confirmatory tasks are independently authored, access-restricted, and screened for sentence, numeric, structural, and semantic overlap. + +## Non-goals + +No provider spend, held-out task publication, hosted dashboard, mutable pricing service, LLM primary grader, routing product, release, or quality claim in this increment. diff --git a/docs/plans/2026-07-17-h1-method-hardening.md b/docs/plans/2026-07-17-h1-method-hardening.md new file mode 100644 index 0000000..e470dd0 --- /dev/null +++ b/docs/plans/2026-07-17-h1-method-hardening.md @@ -0,0 +1,42 @@ +# H1 Method Hardening Implementation Plan + +> **Linear:** DSE-708. Use strict TDD on `feat/h1-pilot-pack`; make zero live provider calls. + +## Phase 1 — Freeze contract and blocked plan + +1. Add failing tests in `tests/evals/test_study_design.py` for complete provenance, two rosters, task-family mapping, cost/spend and policy freezes, tamper detection, exploratory/confirmatory separation, and deterministic task x roster blocked permutations. +2. Extend `src/conclave/evals/models.py` and `protocols.py`; keep old manifests readable only as synthetic exploratory artifacts. +3. Re-run focused tests and commit. + +## Phase 2 — Atomic grading and blinding + +1. Add failing tests for typed atomic errors, severe-error accounting, reviewer seconds, confidence/abstain, grader order/guess, complete double grading, raw agreement/prevalence, undefined constant kappa, and family/roster reliability. +2. Extend `scoring.py`, `blinding.py`, and models. Queue only successful outputs, normalize views, scan leakage, and hash the separate blind map. +3. Re-run focused tests and commit. + +## Phase 3 — Confirmatory estimands and refusal gates + +1. Add tests for task-level roster averaging, task-clustered paired bootstrap, one-sided severe-error and effort/latency noninferiority boundaries, symmetric task exclusions, deviations, cost/latency distributions, and report refusal on freeze drift or exploratory evidence. +2. Extend `scoring.py`, `reporting.py`, and CLI artifact validation. Archive every gate input and seed. +3. Re-run focused tests and commit. + +## Phase 4 — Pilot pack + +1. Add `studies/elite_pilot_v1/public_tasks.json`, embargoed-development `grader_keys.json`, `pilot_protocol.md`, `confirmatory_preregistration.md`, and `README.md`. +2. Add `tests/evals/test_pilot_pack.py`: exactly 24 unique tasks, 12/12 families, 4 per subfamily, balanced tier/readiness, matching nonempty keys, packet citation IDs, no grader material in public records, no duplicates, and documented leakage/holdout rules. +3. Generate and validate an offline manifest only; do not create model outputs. +4. Re-run focused tests and commit. + +## Phase 5 — Verify and hand off + +Run: + +```bash +PYTHONPATH=src PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest -p pytest_asyncio.plugin -p no:cacheprovider -q +python -m ruff check . +python -m ruff format --check . +git diff --check +gitleaks git --redact +``` + +Push, open a draft PR stacked on #52, obtain independent methods/code review, and attach evidence to DSE-708. Paid pilot execution remains blocked until PR #51 is merged/pinned and Ernest approves a hard spend ceiling. From 6b1577a285042f9f934cbb45dbf592911ca1e7c3 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 17:59:55 -0400 Subject: [PATCH 02/13] feat(evals): freeze paid study provenance and blocks --- src/conclave/evals/models.py | 197 ++++++++++++++++++++++- src/conclave/evals/protocols.py | 82 ++++++---- tests/evals/test_study_design.py | 260 +++++++++++++++++++++++++++++++ 3 files changed, 508 insertions(+), 31 deletions(-) create mode 100644 tests/evals/test_study_design.py diff --git a/src/conclave/evals/models.py b/src/conclave/evals/models.py index 81bd18a..21514b1 100644 --- a/src/conclave/evals/models.py +++ b/src/conclave/evals/models.py @@ -2,6 +2,8 @@ from __future__ import annotations +import hashlib +import json from typing import Annotated, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -29,6 +31,7 @@ Sha256Digest = Annotated[str, Field(pattern=r"^sha256:[0-9a-f]{64}$")] RunOutcome = Literal["success", "failed", "timed_out", "malformed", "abstained", "incomplete"] +EvidenceClassification = Literal["synthetic_exploratory", "paid_exploratory_pilot", "confirmatory"] class EvalModel(BaseModel): @@ -63,12 +66,163 @@ class ConditionSpec(EvalModel): description: str = Field(min_length=1) +class ProviderModelSpec(EvalModel): + """One provider/model revision frozen into an experimental roster.""" + + provider_id: str = Field(min_length=1) + model_id: str = Field(min_length=1) + model_revision: str = Field(min_length=1) + + +class RosterSpec(EvalModel): + """A named, immutable provider/model roster.""" + + roster_id: str = Field(min_length=1) + members: tuple[ProviderModelSpec, ...] = Field(min_length=1) + + @model_validator(mode="after") + def validate_unique_members(self) -> RosterSpec: + identities = [(item.provider_id, item.model_id) for item in self.members] + if len(set(identities)) != len(identities): + raise ValueError("roster provider/model identities must be unique") + return self + + +class ExclusionDeviationPolicy(EvalModel): + """Predeclared exclusions while keeping execution deviations in analysis.""" + + predefined_task_exclusions: tuple[str, ...] = () + output_level_exclusions_allowed: Literal[False] = False + deviations_remain_in_denominator: Literal[True] = True + + +class TimeoutRetryPolicy(EvalModel): + """Frozen operational failure policy.""" + + timeout_seconds: float = Field(gt=0) + retry_attempts: int = Field(ge=0) + exhausted_runs_remain_in_denominator: Literal[True] = True + + +class RandomizationConfig(EvalModel): + """Frozen blocked-randomization algorithm and seed.""" + + master_seed: int + method: Literal["sha256_task_roster_block_v1"] = "sha256_task_roster_block_v1" + + +class BootstrapConfig(EvalModel): + """Frozen uncertainty-estimation settings.""" + + seed: int + samples: int = Field(ge=1) + confidence_level: Literal[0.95] = 0.95 + unit: Literal["task"] = "task" + + +class PriceSnapshot(EvalModel): + """Immutable identifier and digest for the price table used by a study.""" + + snapshot_id: str = Field(min_length=1) + captured_at: str = Field(min_length=1) + currency: str = Field(min_length=1) + prices_hash: Sha256Digest + + +class FrozenStudyDesign(EvalModel): + """Complete experimental provenance frozen before any study execution.""" + + evidence_classification: EvidenceClassification + base_commit: str = Field(pattern=r"^[0-9a-f]{40}$") + task_family_map: dict[str, str] = Field(min_length=1) + rosters: tuple[RosterSpec, ...] = Field(min_length=2) + condition_prompt_versions: dict[ConditionId, str] + condition_protocol_versions: dict[ConditionId, str] + generation_settings_hash: Sha256Digest + evaluator_version: str = Field(min_length=1) + analysis_code_hash: Sha256Digest + rubric_hash: Sha256Digest + grader_instructions_hash: Sha256Digest + grader_keys_hash: Sha256Digest + exclusion_deviation_policy: ExclusionDeviationPolicy + timeout_retry_policy: TimeoutRetryPolicy + randomization: RandomizationConfig + bootstrap: BootstrapConfig + price_snapshot: PriceSnapshot + approved_spend_ceiling_usd: float = Field(ge=0) + preregistration_id: str | None = None + preregistration_hash: Sha256Digest | None = None + + @model_validator(mode="after") + def validate_freeze(self) -> FrozenStudyDesign: + roster_ids = [roster.roster_id for roster in self.rosters] + if len(set(roster_ids)) != len(roster_ids): + raise ValueError("frozen roster_id values must be unique") + expected_conditions = set(EVAL_CONDITION_IDS) + if set(self.condition_prompt_versions) != expected_conditions: + raise ValueError("condition_prompt_versions must cover exactly all conditions") + if set(self.condition_protocol_versions) != expected_conditions: + raise ValueError("condition_protocol_versions must cover exactly all conditions") + if any(not value for value in self.condition_prompt_versions.values()): + raise ValueError("condition prompt versions must be nonempty") + if any(not value for value in self.condition_protocol_versions.values()): + raise ValueError("condition protocol versions must be nonempty") + if any(not value for value in self.task_family_map.values()): + raise ValueError("task family values must be nonempty") + if self.evidence_classification in {"paid_exploratory_pilot", "confirmatory"}: + if self.approved_spend_ceiling_usd <= 0: + raise ValueError("paid studies require a positive approved spend ceiling") + if self.evidence_classification == "confirmatory" and ( + not self.preregistration_id or self.preregistration_hash is None + ): + raise ValueError("confirmatory studies require preregistration ID and hash") + return self + + +def hash_frozen_study_design(design: FrozenStudyDesign) -> str: + """Return a canonical digest that detects any frozen-design drift.""" + + canonical = json.dumps( + design.model_dump(mode="json"), sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" + + +def derive_planned_run_id( + *, + study_id: str, + task_id: str, + condition_id: ConditionId, + replicate: int, + max_output_tokens: int, + roster_id: str = "legacy_default", + frozen_design_hash: str | None = None, +) -> str: + """Derive the canonical cell identity, preserving the legacy payload exactly.""" + + fields: dict[str, str | int] = { + "schema_version": EVAL_SCHEMA_VERSION, + "study_id": study_id, + "task_id": task_id, + "condition_id": condition_id, + "replicate": replicate, + "max_output_tokens": max_output_tokens, + } + if roster_id != "legacy_default": + fields["roster_id"] = roster_id + if frozen_design_hash is not None: + fields["frozen_design_hash"] = frozen_design_hash + identity = json.dumps(fields, sort_keys=True, separators=(",", ":")).encode("utf-8") + return f"run_{hashlib.sha256(identity).hexdigest()[:24]}" + + class PlannedRun(EvalModel): """An immutable cell declared before any model execution.""" planned_run_id: str = Field(pattern=r"^run_[0-9a-f]{24}$") study_id: str = Field(min_length=1) task_id: str = Field(min_length=1) + roster_id: str = Field(default="legacy_default", min_length=1) condition_id: ConditionId replicate: int = Field(ge=1) max_output_tokens: int = Field(gt=0) @@ -120,10 +274,14 @@ class StudyManifest(EvalModel): """Frozen preregistration of the complete execution matrix.""" study_id: str = Field(min_length=1) + evidence_classification: EvidenceClassification = "synthetic_exploratory" + promotable: Literal[False] = False seed: int replicates: int = Field(ge=1) task_ids: tuple[str, ...] = Field(min_length=1) public_tasks_hash: Sha256Digest + frozen_design: FrozenStudyDesign | None = None + frozen_design_hash: Sha256Digest | None = None planned_runs: tuple[PlannedRun, ...] = Field(min_length=1) @model_validator(mode="after") @@ -131,21 +289,54 @@ def validate_complete_matrix(self) -> StudyManifest: if len(set(self.task_ids)) != len(self.task_ids): raise ValueError("task_ids must be unique") + if self.frozen_design is None: + if self.evidence_classification != "synthetic_exploratory": + raise ValueError("a legacy manifest classification must be synthetic exploratory") + if self.frozen_design_hash is not None: + raise ValueError("a design hash requires a frozen design") + roster_ids = ("legacy_default",) + else: + if self.evidence_classification != self.frozen_design.evidence_classification: + raise ValueError("manifest and frozen design classification must match") + if self.frozen_design_hash != hash_frozen_study_design(self.frozen_design): + raise ValueError("frozen design hash does not match its contents") + if set(self.frozen_design.task_family_map) != set(self.task_ids): + raise ValueError("task_family_map must exactly cover manifest task_ids") + if self.seed != self.frozen_design.randomization.master_seed: + raise ValueError("manifest seed must match the frozen randomization master seed") + roster_ids = tuple(roster.roster_id for roster in self.frozen_design.rosters) + expected = { - (task_id, condition_id, replicate) + (task_id, roster_id, condition_id, replicate) for task_id in self.task_ids + for roster_id in roster_ids for condition_id in EVAL_CONDITION_IDS for replicate in range(1, self.replicates + 1) } - actual = {(run.task_id, run.condition_id, run.replicate) for run in self.planned_runs} + actual = { + (run.task_id, run.roster_id, run.condition_id, run.replicate) + for run in self.planned_runs + } if actual != expected or len(self.planned_runs) != len(expected): raise ValueError( - "planned_runs must contain the complete task x condition x replicate matrix" + "planned_runs must contain the complete task x roster x condition x replicate matrix" ) if any(run.study_id != self.study_id for run in self.planned_runs): raise ValueError("every planned run must belong to this study") if len({run.planned_run_id for run in self.planned_runs}) != len(self.planned_runs): raise ValueError("planned_run_id values must be unique") + for run in self.planned_runs: + expected_id = derive_planned_run_id( + study_id=run.study_id, + task_id=run.task_id, + roster_id=run.roster_id, + condition_id=run.condition_id, + replicate=run.replicate, + max_output_tokens=run.max_output_tokens, + frozen_design_hash=self.frozen_design_hash, + ) + if run.planned_run_id != expected_id: + raise ValueError(f"planned_run_id does not match frozen cell: {run.planned_run_id}") return self diff --git a/src/conclave/evals/protocols.py b/src/conclave/evals/protocols.py index 221ec1f..e4ca919 100644 --- a/src/conclave/evals/protocols.py +++ b/src/conclave/evals/protocols.py @@ -10,12 +10,14 @@ from .dataset import hash_public_tasks from .models import ( EVAL_CONDITION_IDS, - EVAL_SCHEMA_VERSION, ConditionId, ConditionSpec, + FrozenStudyDesign, PlannedRun, PublicTask, StudyManifest, + derive_planned_run_id, + hash_frozen_study_design, ) CONDITIONS: tuple[ConditionSpec, ...] = ( @@ -42,6 +44,27 @@ def condition_order(seed: int) -> tuple[ConditionId, ...]: return tuple(ordered) +def blocked_condition_order( + *, master_seed: int, task_id: str, roster_id: str +) -> tuple[ConditionId, ...]: + """Derive one deterministic, independent permutation per task-roster block.""" + + identity = json.dumps( + { + "method": "sha256_task_roster_block_v1", + "master_seed": master_seed, + "task_id": task_id, + "roster_id": roster_id, + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + derived_seed = int.from_bytes(hashlib.sha256(identity).digest(), "big") + ordered = list(CONDITION_IDS) + random.Random(derived_seed).shuffle(ordered) + return tuple(ordered) + + def _validate_budgets(output_token_budgets: Mapping[str, int]) -> None: if set(output_token_budgets) != set(CONDITION_IDS): raise ValueError("output_token_budgets must contain exactly the six frozen conditions") @@ -56,29 +79,6 @@ def _validate_budgets(output_token_budgets: Mapping[str, int]) -> None: raise ValueError("planned output-token ceilings must remain within 5% across conditions") -def _planned_run_id( - *, - study_id: str, - task_id: str, - condition_id: ConditionId, - replicate: int, - max_output_tokens: int, -) -> str: - identity = json.dumps( - { - "schema_version": EVAL_SCHEMA_VERSION, - "study_id": study_id, - "task_id": task_id, - "condition_id": condition_id, - "replicate": replicate, - "max_output_tokens": max_output_tokens, - }, - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") - return f"run_{hashlib.sha256(identity).hexdigest()[:24]}" - - def build_study_manifest( *, study_id: str, @@ -86,8 +86,9 @@ def build_study_manifest( replicates: int, seed: int, output_token_budgets: Mapping[str, int], + frozen_design: FrozenStudyDesign | None = None, ) -> StudyManifest: - """Predeclare every task-condition-replicate cell with stable identities.""" + """Predeclare every legacy or frozen task-roster-condition-replicate cell.""" if replicates < 1: raise ValueError("replicates must be at least one") @@ -97,33 +98,58 @@ def build_study_manifest( if len(set(task_ids)) != len(task_ids): raise ValueError("public task_id values must be unique") _validate_budgets(output_token_budgets) + if frozen_design is not None: + if set(frozen_design.task_family_map) != set(task_ids): + raise ValueError("task_family_map must exactly cover public task IDs") + if frozen_design.randomization.master_seed != seed: + raise ValueError("seed must match the frozen randomization master seed") sorted_tasks = sorted(tasks, key=lambda task: task.task_id) - ordered_conditions = condition_order(seed) + roster_ids = ( + tuple(sorted(roster.roster_id for roster in frozen_design.rosters)) + if frozen_design is not None + else ("legacy_default",) + ) + design_hash = hash_frozen_study_design(frozen_design) if frozen_design is not None else None planned_runs = tuple( PlannedRun( - planned_run_id=_planned_run_id( + planned_run_id=derive_planned_run_id( study_id=study_id, task_id=task.task_id, condition_id=condition_id, replicate=replicate, max_output_tokens=output_token_budgets[condition_id], + roster_id=roster_id, + frozen_design_hash=design_hash, ), study_id=study_id, task_id=task.task_id, + roster_id=roster_id, condition_id=condition_id, replicate=replicate, max_output_tokens=output_token_budgets[condition_id], ) for task in sorted_tasks for replicate in range(1, replicates + 1) - for condition_id in ordered_conditions + for roster_id in roster_ids + for condition_id in ( + blocked_condition_order(master_seed=seed, task_id=task.task_id, roster_id=roster_id) + if frozen_design is not None + else condition_order(seed) + ) ) return StudyManifest( study_id=study_id, + evidence_classification=( + frozen_design.evidence_classification + if frozen_design is not None + else "synthetic_exploratory" + ), seed=seed, replicates=replicates, task_ids=tuple(task.task_id for task in sorted_tasks), public_tasks_hash=hash_public_tasks(sorted_tasks), + frozen_design=frozen_design, + frozen_design_hash=design_hash, planned_runs=planned_runs, ) diff --git a/tests/evals/test_study_design.py b/tests/evals/test_study_design.py new file mode 100644 index 0000000..6c75dc9 --- /dev/null +++ b/tests/evals/test_study_design.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import json + +import pytest +from pydantic import ValidationError + +from conclave.evals.models import ( + BootstrapConfig, + ExclusionDeviationPolicy, + FrozenStudyDesign, + PriceSnapshot, + ProviderModelSpec, + PublicTask, + RandomizationConfig, + RosterSpec, + StudyManifest, + TimeoutRetryPolicy, +) +from conclave.evals.protocols import ( + CONDITION_IDS, + blocked_condition_order, + build_study_manifest, +) + +DIGEST = "sha256:" + "a" * 64 + + +def _design( + *, + evidence_classification="paid_exploratory_pilot", + preregistration_id=None, + preregistration_hash=None, + rosters=None, +): + rosters = rosters or ( + RosterSpec( + roster_id="roster-a", + members=( + ProviderModelSpec( + provider_id="provider-a", + model_id="model-a", + model_revision="2026-07-01", + ), + ), + ), + RosterSpec( + roster_id="roster-b", + members=( + ProviderModelSpec( + provider_id="provider-b", + model_id="model-b", + model_revision="2026-07-02", + ), + ), + ), + ) + return FrozenStudyDesign( + evidence_classification=evidence_classification, + base_commit="1" * 40, + task_family_map={"task-a": "operational", "task-b": "stewardship"}, + rosters=rosters, + condition_prompt_versions={condition: "prompt-v1" for condition in CONDITION_IDS}, + condition_protocol_versions={condition: "protocol-v1" for condition in CONDITION_IDS}, + generation_settings_hash=DIGEST, + evaluator_version="evaluator-v1", + analysis_code_hash=DIGEST, + rubric_hash=DIGEST, + grader_instructions_hash=DIGEST, + grader_keys_hash=DIGEST, + exclusion_deviation_policy=ExclusionDeviationPolicy( + predefined_task_exclusions=("duplicate", "reference-invalid"), + ), + timeout_retry_policy=TimeoutRetryPolicy(timeout_seconds=120, retry_attempts=1), + randomization=RandomizationConfig(master_seed=20260717), + bootstrap=BootstrapConfig(seed=991, samples=10000), + price_snapshot=PriceSnapshot( + snapshot_id="prices-2026-07-17", + captured_at="2026-07-17T12:00:00Z", + currency="USD", + prices_hash=DIGEST, + ), + approved_spend_ceiling_usd=250.0, + preregistration_id=preregistration_id, + preregistration_hash=preregistration_hash, + ) + + +def _tasks(): + return [ + PublicTask(task_id="task-b", prompt="B"), + PublicTask(task_id="task-a", prompt="A"), + ] + + +def test_frozen_design_captures_complete_provenance_and_is_tamper_evident() -> None: + manifest = build_study_manifest( + study_id="paid-pilot", + tasks=_tasks(), + replicates=1, + seed=20260717, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 1000), + frozen_design=_design(), + ) + payload = manifest.model_dump(mode="json") + + assert manifest.evidence_classification == "paid_exploratory_pilot" + assert manifest.promotable is False + assert manifest.frozen_design_hash.startswith("sha256:") + assert payload["frozen_design"]["base_commit"] == "1" * 40 + assert payload["frozen_design"]["grader_keys_hash"] == DIGEST + assert payload["frozen_design"]["approved_spend_ceiling_usd"] == 250.0 + assert payload["frozen_design"]["exclusion_deviation_policy"] + assert payload["frozen_design"]["timeout_retry_policy"] + assert payload["frozen_design"]["price_snapshot"] + + tampered = json.loads(manifest.model_dump_json()) + tampered["frozen_design"]["evaluator_version"] = "changed-after-freeze" + with pytest.raises(ValidationError, match="frozen design hash"): + StudyManifest.model_validate(tampered) + + changed_rosters = list(_design().rosters) + changed_rosters[0] = changed_rosters[0].model_copy( + update={ + "members": ( + changed_rosters[0].members[0].model_copy(update={"model_revision": "2026-07-18"}), + ) + } + ) + changed = build_study_manifest( + study_id="paid-pilot", + tasks=_tasks(), + replicates=1, + seed=20260717, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 1000), + frozen_design=_design(rosters=tuple(changed_rosters)), + ) + assert {run.planned_run_id for run in changed.planned_runs}.isdisjoint( + run.planned_run_id for run in manifest.planned_runs + ) + + tampered_run_id = json.loads(manifest.model_dump_json()) + tampered_run_id["planned_runs"][0]["planned_run_id"] = "run_" + "f" * 24 + with pytest.raises(ValidationError, match="planned_run_id"): + StudyManifest.model_validate(tampered_run_id) + + +def test_confirmatory_requires_preregistration_and_every_freeze_field() -> None: + with pytest.raises(ValidationError, match="preregistration"): + _design(evidence_classification="confirmatory") + + design = _design( + evidence_classification="confirmatory", + preregistration_id="osf:conclave-confirmatory-v1", + preregistration_hash=DIGEST, + ) + manifest = build_study_manifest( + study_id="confirmatory-v1", + tasks=_tasks(), + replicates=1, + seed=20260717, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 1000), + frozen_design=design, + ) + + assert manifest.evidence_classification == "confirmatory" + assert manifest.frozen_design.preregistration_hash == DIGEST + + +def test_exploratory_manifest_cannot_be_promoted_after_creation() -> None: + manifest = build_study_manifest( + study_id="exploratory", + tasks=_tasks(), + replicates=1, + seed=20260717, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 1000), + frozen_design=_design(evidence_classification="synthetic_exploratory"), + ) + payload = json.loads(manifest.model_dump_json()) + payload["evidence_classification"] = "confirmatory" + + with pytest.raises(ValidationError, match="classification"): + StudyManifest.model_validate(payload) + with pytest.raises(ValidationError): + StudyManifest.model_validate({**json.loads(manifest.model_dump_json()), "promotable": True}) + + +def test_design_requires_two_unique_rosters_and_exact_task_family_coverage() -> None: + one_roster = ( + RosterSpec( + roster_id="only", + members=( + ProviderModelSpec( + provider_id="provider", model_id="model", model_revision="revision" + ), + ), + ), + ) + with pytest.raises(ValidationError, match="at least 2"): + _design(rosters=one_roster) + + design = _design().model_copy( + update={"task_family_map": {"task-a": "operational", "extra": "wrong"}} + ) + with pytest.raises(ValueError, match="task_family_map"): + build_study_manifest( + study_id="bad-families", + tasks=_tasks(), + replicates=1, + seed=20260717, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 1000), + frozen_design=design, + ) + + +def test_blocked_plan_is_stable_complete_and_independent_per_task_roster() -> None: + design = _design() + kwargs = dict( + study_id="blocked-pilot", + replicates=2, + seed=20260717, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 1000), + frozen_design=design, + ) + first = build_study_manifest(tasks=_tasks(), **kwargs) + second = build_study_manifest(tasks=list(reversed(_tasks())), **kwargs) + + assert first == second + assert len(first.planned_runs) == 2 * 2 * 6 * 2 + assert {run.roster_id for run in first.planned_runs} == {"roster-a", "roster-b"} + assert len({run.planned_run_id for run in first.planned_runs}) == 48 + for task_id in ("task-a", "task-b"): + for roster_id in ("roster-a", "roster-b"): + block = tuple( + run.condition_id + for run in first.planned_runs + if run.task_id == task_id and run.roster_id == roster_id and run.replicate == 1 + ) + assert block == blocked_condition_order( + master_seed=20260717, task_id=task_id, roster_id=roster_id + ) + assert blocked_condition_order( + master_seed=20260717, task_id="task-a", roster_id="roster-a" + ) != blocked_condition_order(master_seed=20260717, task_id="task-b", roster_id="roster-b") + + +def test_legacy_builder_remains_synthetic_exploratory_and_readable() -> None: + manifest = build_study_manifest( + study_id="legacy", + tasks=[PublicTask(task_id="task-a", prompt="A")], + replicates=1, + seed=7, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 1000), + ) + round_trip = StudyManifest.model_validate_json(manifest.model_dump_json()) + + assert round_trip == manifest + assert manifest.evidence_classification == "synthetic_exploratory" + assert manifest.frozen_design is None + assert {run.roster_id for run in manifest.planned_runs} == {"legacy_default"} From 8038e9df6525af426b139eac0e20d73cb8bac70f Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:07:57 -0400 Subject: [PATCH 03/13] feat(evals): harden paid grading and blinding --- src/conclave/evals/blinding.py | 71 +++++++- src/conclave/evals/scoring.py | 200 ++++++++++++++++++++--- tests/evals/test_blinding.py | 46 +++++- tests/evals/test_method_grading.py | 251 +++++++++++++++++++++++++++++ 4 files changed, 547 insertions(+), 21 deletions(-) create mode 100644 tests/evals/test_method_grading.py diff --git a/src/conclave/evals/blinding.py b/src/conclave/evals/blinding.py index 42a0163..ab9b5ca 100644 --- a/src/conclave/evals/blinding.py +++ b/src/conclave/evals/blinding.py @@ -3,12 +3,13 @@ from __future__ import annotations import hashlib +import json import random from collections.abc import Sequence -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator -from .models import EVAL_SCHEMA_VERSION, RunOutcome, RunRecord, SchemaVersion +from .models import EVAL_SCHEMA_VERSION, RunOutcome, RunRecord, SchemaVersion, Sha256Digest class BlindModel(BaseModel): @@ -43,6 +44,37 @@ class BlindMap(BlindModel): """Separate restricted identity artifact, never included in grader output.""" entries: tuple[BlindMapEntry, ...] + blind_map_hash: Sha256Digest | None = None + + @model_validator(mode="after") + def validate_hash(self) -> BlindMap: + if self.blind_map_hash is not None and self.blind_map_hash != hash_blind_map(self.entries): + raise ValueError("blind map hash does not match its entries") + return self + + +class GraderQueueOutput(BlindModel): + """One normalized successful output safe for paid human grading.""" + + opaque_output_id: str = Field(pattern=r"^output_[0-9a-f]{24}$") + presentation: str = Field(min_length=1) + + +class GraderQueue(BlindModel): + """Successful-only grader queue with no execution labels or outcomes.""" + + outputs: tuple[GraderQueueOutput, ...] + + +def hash_blind_map(entries: Sequence[BlindMapEntry]) -> str: + """Hash a restricted identity map in its frozen queue order.""" + + canonical = json.dumps( + [entry.model_dump(mode="json") for entry in entries], + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" def _opaque_id(*, planned_run_id: str, seed: int) -> str: @@ -76,3 +108,38 @@ def blind_run_records( BlindedOutputSet(outputs=tuple(output for output, _ in blinded)), BlindMap(entries=tuple(entry for _, entry in blinded)), ) + + +def build_grader_queue( + records: Sequence[RunRecord], *, seed: int, forbidden_labels: Sequence[str] +) -> tuple[GraderQueue, BlindMap]: + """Normalize, leak-check, and blind successful outputs for paid grading.""" + + if len({record.planned_run_id for record in records}) != len(records): + raise ValueError("run records must have unique planned_run_id values") + blinded = [] + labels = tuple(label.casefold() for label in forbidden_labels if label) + for record in records: + if record.outcome != "success": + continue + if record.output is None: + raise ValueError("successful grader-queue records require output") + presentation = " ".join(record.output.split()) + if not presentation: + raise ValueError("successful grader-queue records require nonempty output") + folded = presentation.casefold() + if any(label in folded for label in labels): + raise ValueError("grader presentation contains a forbidden label") + opaque_id = _opaque_id(planned_run_id=record.planned_run_id, seed=seed) + blinded.append( + ( + GraderQueueOutput(opaque_output_id=opaque_id, presentation=presentation), + BlindMapEntry(opaque_output_id=opaque_id, planned_run_id=record.planned_run_id), + ) + ) + random.Random(seed).shuffle(blinded) + entries = tuple(entry for _, entry in blinded) + return ( + GraderQueue(outputs=tuple(output for output, _ in blinded)), + BlindMap(entries=entries, blind_map_hash=hash_blind_map(entries)), + ) diff --git a/src/conclave/evals/scoring.py b/src/conclave/evals/scoring.py index f43037d..462076b 100644 --- a/src/conclave/evals/scoring.py +++ b/src/conclave/evals/scoring.py @@ -6,6 +6,7 @@ import random from collections import Counter, defaultdict from collections.abc import Mapping, Sequence +from typing import Literal from pydantic import Field, model_validator @@ -14,12 +15,34 @@ ConditionId, EvalModel, RunOutcome, + Sha256Digest, StudyManifest, StudyRun, ) from .runner import validate_run_records +class AtomicError(EvalModel): + """One rubric-linked error identified in a grader-visible output.""" + + rubric_item: str = Field(min_length=1) + category: str = Field(min_length=1) + severity: Literal["minor", "major", "severe"] + detail: str = Field(min_length=1) + evidence_packet_ids: tuple[str, ...] = () + + +class RubricDimensions(EvalModel): + """Fixed 0–2 holistic rubric dimensions.""" + + constraint_recall: int = Field(ge=0, le=2) + conflict_minority_recognition: int = Field(ge=0, le=2) + unsupported_claim_avoidance: int = Field(ge=0, le=2) + recommendation_correctness: int = Field(ge=0, le=2) + completeness_actionability: int = Field(ge=0, le=2) + readiness_calibration: int = Field(ge=0, le=2) + + class GraderJudgment(EvalModel): """One immutable raw grader judgment tied to one blinded or planned output.""" @@ -30,11 +53,33 @@ class GraderJudgment(EvalModel): critical_error_free: bool dimensions: dict[str, int] = Field(default_factory=dict) notes: str | None = None + atomic_errors: tuple[AtomicError, ...] = () + severe_error: bool | None = None + rubric_dimensions: RubricDimensions | None = None + reviewer_seconds: float | None = Field(default=None, gt=0) + confidence: Literal["low", "medium", "high"] | None = None + abstained: bool | None = None + rubric_version: str | None = None + rubric_hash: Sha256Digest | None = None + grader_batch: str | None = None + grader_order: int | None = Field(default=None, ge=1) + condition_guess: ConditionId | Literal["unknown"] | None = None + provider_guess: str | None = None @model_validator(mode="after") def validate_one_target(self) -> GraderJudgment: if (self.planned_run_id is None) == (self.opaque_output_id is None): raise ValueError("exactly one of planned_run_id or opaque_output_id is required") + has_severe = any(error.severity == "severe" for error in self.atomic_errors) + if self.severe_error is not None and self.severe_error != has_severe: + raise ValueError("severe_error must match the atomic error severities") + has_disqualifying_error = any( + error.severity in {"major", "severe"} for error in self.atomic_errors + ) + if self.critical_error_free and has_disqualifying_error: + raise ValueError("critical_error_free must be false for major or severe errors") + if self.abstained is True and self.critical_error_free: + raise ValueError("abstained judgments cannot be critical_error_free") return self @@ -108,6 +153,20 @@ class ReliabilityMetrics(EvalModel): disagreements: int = Field(ge=0) adjudicated_disagreements: int = Field(ge=0) adjudication_rate: float = Field(ge=0.0, le=1.0) + raw_agreement: float | None = Field(default=None, ge=0.0, le=1.0) + positive_prevalence: float | None = Field(default=None, ge=0.0, le=1.0) + strata: tuple[ReliabilityStratum, ...] = () + + +class ReliabilityStratum(EvalModel): + """Agreement statistics for one frozen family or roster stratum.""" + + stratum_type: Literal["family", "roster"] + stratum_value: str = Field(min_length=1) + paired_judgments: int = Field(ge=0) + raw_agreement: float | None = Field(default=None, ge=0.0, le=1.0) + positive_prevalence: float | None = Field(default=None, ge=0.0, le=1.0) + cohen_kappa: float | None = Field(default=None, ge=-1.0, le=1.0) class StudyScoreReport(EvalModel): @@ -214,7 +273,7 @@ def cohen_kappa(first: Sequence[bool], second: Sequence[bool]) -> float | None: second_true = sum(second) / len(second) expected = first_true * second_true + (1 - first_true) * (1 - second_true) if expected == 1.0: - return 1.0 if observed == 1.0 else None + return None return (observed - expected) / (1 - expected) @@ -237,11 +296,38 @@ def _planned_target(record: GraderJudgment | AdjudicationRecord, mapping: Mappin raise ValueError("opaque target is missing from the blind map") from exc +def _paired_values( + judgments_by_run: Mapping[str, Sequence[GraderJudgment]], run_ids: Sequence[str] +) -> tuple[tuple[str, str] | None, list[tuple[bool, bool]]]: + graders = sorted({item.grader_id for values in judgments_by_run.values() for item in values}) + if len(graders) != 2: + return None, [] + first, second = graders + pairs = [] + for run_id in sorted(run_ids): + by_grader = {item.grader_id: item.critical_error_free for item in judgments_by_run[run_id]} + if first in by_grader and second in by_grader: + pairs.append((by_grader[first], by_grader[second])) + return (first, second), pairs + + +def _agreement_statistics( + pairs: Sequence[tuple[bool, bool]], +) -> tuple[float | None, float | None, float | None]: + if not pairs: + return None, None, None + first = [left for left, _ in pairs] + second = [right for _, right in pairs] + raw_agreement = sum(left == right for left, right in pairs) / len(pairs) + prevalence = (sum(first) + sum(second)) / (2 * len(pairs)) + return raw_agreement, prevalence, cohen_kappa(first, second) + + def _reliability( judgments_by_run: Mapping[str, Sequence[GraderJudgment]], adjudicated_runs: set[str], + manifest: StudyManifest, ) -> ReliabilityMetrics: - graders = sorted({item.grader_id for values in judgments_by_run.values() for item in values}) disagreements = sum( len({item.critical_error_free for item in values}) > 1 for values in judgments_by_run.values() @@ -252,29 +338,102 @@ def _reliability( if len({item.critical_error_free for item in values}) > 1 ) rate = adjudicated_disagreements / disagreements if disagreements else 0.0 - if len(graders) != 2: - return ReliabilityMetrics( - paired_judgments=0, - disagreements=disagreements, - adjudicated_disagreements=adjudicated_disagreements, - adjudication_rate=rate, - ) - first, second = graders - pairs = [] - for run_id in sorted(judgments_by_run): - by_grader = {item.grader_id: item.critical_error_free for item in judgments_by_run[run_id]} - if first in by_grader and second in by_grader: - pairs.append((by_grader[first], by_grader[second])) + grader_pair, pairs = _paired_values(judgments_by_run, tuple(judgments_by_run)) + raw_agreement, prevalence, kappa = _agreement_statistics(pairs) + strata: list[ReliabilityStratum] = [] + if manifest.frozen_design is not None: + for family in sorted(set(manifest.frozen_design.task_family_map.values())): + run_ids = [ + run.planned_run_id + for run in manifest.planned_runs + if manifest.frozen_design.task_family_map[run.task_id] == family + ] + _, stratum_pairs = _paired_values(judgments_by_run, run_ids) + agreement, stratum_prevalence, stratum_kappa = _agreement_statistics(stratum_pairs) + strata.append( + ReliabilityStratum( + stratum_type="family", + stratum_value=family, + paired_judgments=len(stratum_pairs), + raw_agreement=agreement, + positive_prevalence=stratum_prevalence, + cohen_kappa=stratum_kappa, + ) + ) + for roster_id in sorted(roster.roster_id for roster in manifest.frozen_design.rosters): + run_ids = [ + run.planned_run_id for run in manifest.planned_runs if run.roster_id == roster_id + ] + _, stratum_pairs = _paired_values(judgments_by_run, run_ids) + agreement, stratum_prevalence, stratum_kappa = _agreement_statistics(stratum_pairs) + strata.append( + ReliabilityStratum( + stratum_type="roster", + stratum_value=roster_id, + paired_judgments=len(stratum_pairs), + raw_agreement=agreement, + positive_prevalence=stratum_prevalence, + cohen_kappa=stratum_kappa, + ) + ) return ReliabilityMetrics( - grader_pair=(first, second), - cohen_kappa=cohen_kappa([left for left, _ in pairs], [right for _, right in pairs]), + grader_pair=grader_pair, + cohen_kappa=kappa, paired_judgments=len(pairs), disagreements=disagreements, adjudicated_disagreements=adjudicated_disagreements, adjudication_rate=rate, + raw_agreement=raw_agreement, + positive_prevalence=prevalence, + strata=tuple(strata), ) +def _validate_paid_grading( + *, + manifest: StudyManifest, + study_run: StudyRun, + judgments_by_run: Mapping[str, Sequence[GraderJudgment]], +) -> None: + """Fail closed on incomplete or non-independent paid-study grading.""" + + if manifest.evidence_classification == "synthetic_exploratory": + return + run_by_id = {record.planned_run_id: record for record in study_run.records} + required_fields = ( + "severe_error", + "rubric_dimensions", + "reviewer_seconds", + "confidence", + "abstained", + "rubric_version", + "rubric_hash", + "grader_batch", + "grader_order", + "condition_guess", + "provider_guess", + ) + for run_id, judgments in judgments_by_run.items(): + if run_by_id[run_id].outcome != "success": + raise ValueError("non-success cells must remain outside paid human grading") + if any( + getattr(judgment, field) is None for judgment in judgments for field in required_fields + ): + raise ValueError("paid judgments require complete paid-study fields") + if manifest.frozen_design is not None and any( + judgment.rubric_hash != manifest.frozen_design.rubric_hash for judgment in judgments + ): + raise ValueError("paid judgment rubric hash must match the frozen design") + for record in study_run.records: + if record.outcome != "success": + continue + judgments = judgments_by_run[record.planned_run_id] + if len(judgments) != 2 or len({item.grader_id for item in judgments}) != 2: + raise ValueError( + "every successful paid cell requires exactly two independent judgments" + ) + + def score_study( *, manifest: StudyManifest, @@ -313,6 +472,10 @@ def score_study( seen_grader_targets.add(grader_target) judgments_by_run[run_id].append(judgment) + _validate_paid_grading( + manifest=manifest, study_run=study_run, judgments_by_run=judgments_by_run + ) + adjudication_by_run: dict[str, AdjudicationRecord] = {} for adjudication in adjudications: run_id = _planned_target(adjudication, mapping) @@ -399,13 +562,14 @@ def score_study( ) return StudyScoreReport( study_id=manifest.study_id, + evidence_classification=manifest.evidence_classification, total_planned_runs=len(manifest.planned_runs), run_outcome_distribution=dict( sorted(Counter(item.outcome for item in study_run.records).items()) ), condition_metrics=tuple(condition_metrics), paired_differences=paired, - reliability=_reliability(judgments_by_run, set(adjudication_by_run)), + reliability=_reliability(judgments_by_run, set(adjudication_by_run), manifest), raw_judgments=tuple(raw_judgments), adjudications=tuple(adjudications), resolved_outcomes=tuple(resolved), diff --git a/tests/evals/test_blinding.py b/tests/evals/test_blinding.py index a80e431..df9a2c6 100644 --- a/tests/evals/test_blinding.py +++ b/tests/evals/test_blinding.py @@ -1,6 +1,11 @@ from __future__ import annotations -from conclave.evals.blinding import blind_run_records +import json + +import pytest +from pydantic import ValidationError + +from conclave.evals.blinding import BlindMap, blind_run_records, build_grader_queue from conclave.evals.models import RunRecord @@ -42,3 +47,42 @@ def test_blinded_outputs_exclude_run_condition_provider_and_model_labels() -> No assert "run_aaaaaaaaaaaaaaaaaaaaaaaa" not in serialized_outputs assert "Alpha" in serialized_outputs assert "planned_run_id" in blind_map.model_dump_json() + + +def test_paid_grader_queue_contains_only_normalized_successes_and_hashed_map() -> None: + queue, blind_map = build_grader_queue( + _records(), seed=31, forbidden_labels=("elite_full", "provider-a", "model-a") + ) + + assert len(queue.outputs) == 1 + assert queue.outputs[0].presentation == "Alpha" + assert blind_map.blind_map_hash.startswith("sha256:") + assert "outcome" not in queue.model_dump_json() + assert "planned_run_id" not in queue.model_dump_json() + + tampered = json.loads(blind_map.model_dump_json()) + tampered["entries"][0]["planned_run_id"] = "run_" + "f" * 24 + with pytest.raises(ValidationError, match="blind map hash"): + BlindMap.model_validate(tampered) + + +def test_paid_grader_queue_normalizes_formatting_and_rejects_label_leakage() -> None: + records = ( + RunRecord( + planned_run_id="run_" + "d" * 24, + outcome="success", + output=" First\r\n\r\n second line ", + ), + ) + queue, _ = build_grader_queue(records, seed=4, forbidden_labels=("provider-a",)) + assert queue.outputs[0].presentation == "First second line" + + leaked = ( + RunRecord( + planned_run_id="run_" + "e" * 24, + outcome="success", + output="Generated by Provider-A", + ), + ) + with pytest.raises(ValueError, match="forbidden label"): + build_grader_queue(leaked, seed=4, forbidden_labels=("provider-a",)) diff --git a/tests/evals/test_method_grading.py b/tests/evals/test_method_grading.py new file mode 100644 index 0000000..ebeba5c --- /dev/null +++ b/tests/evals/test_method_grading.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from conclave.evals.models import ( + BootstrapConfig, + ExclusionDeviationPolicy, + FrozenStudyDesign, + PriceSnapshot, + ProviderModelSpec, + PublicTask, + RandomizationConfig, + RosterSpec, + RunRecord, + StudyRun, + TimeoutRetryPolicy, +) +from conclave.evals.protocols import CONDITION_IDS, build_study_manifest +from conclave.evals.scoring import ( + AtomicError, + GraderJudgment, + RubricDimensions, + cohen_kappa, + score_study, +) + +DIGEST = "sha256:" + "a" * 64 + + +def _paid_study(): + task = PublicTask(task_id="task-a", prompt="Choose.") + design = FrozenStudyDesign( + evidence_classification="paid_exploratory_pilot", + base_commit="1" * 40, + task_family_map={"task-a": "operational"}, + rosters=tuple( + RosterSpec( + roster_id=f"roster-{suffix}", + members=( + ProviderModelSpec( + provider_id=f"provider-{suffix}", + model_id=f"model-{suffix}", + model_revision="v1", + ), + ), + ) + for suffix in ("a", "b") + ), + condition_prompt_versions=dict.fromkeys(CONDITION_IDS, "prompt-v1"), + condition_protocol_versions=dict.fromkeys(CONDITION_IDS, "protocol-v1"), + generation_settings_hash=DIGEST, + evaluator_version="eval-v1", + analysis_code_hash=DIGEST, + rubric_hash=DIGEST, + grader_instructions_hash=DIGEST, + grader_keys_hash=DIGEST, + exclusion_deviation_policy=ExclusionDeviationPolicy(), + timeout_retry_policy=TimeoutRetryPolicy(timeout_seconds=60, retry_attempts=0), + randomization=RandomizationConfig(master_seed=17), + bootstrap=BootstrapConfig(seed=19, samples=100), + price_snapshot=PriceSnapshot( + snapshot_id="prices-v1", + captured_at="2026-07-17T00:00:00Z", + currency="USD", + prices_hash=DIGEST, + ), + approved_spend_ceiling_usd=10, + ) + manifest = build_study_manifest( + study_id="paid", + tasks=[task], + replicates=1, + seed=17, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 100), + frozen_design=design, + ) + records = tuple( + RunRecord( + planned_run_id=planned.planned_run_id, + outcome="failed" if index == 0 else "success", + output=None if index == 0 else "answer", + error_category="executor_error" if index == 0 else None, + ) + for index, planned in enumerate(manifest.planned_runs) + ) + study_run = StudyRun( + study_id=manifest.study_id, + records=records, + total_planned_runs=len(records), + outcome_counts={"failed": 1, "success": len(records) - 1}, + total_completion_tokens=0, + total_latency_ms=0, + ) + return manifest, study_run + + +def _judgment(run_id: str, grader: str, *, value: bool = True) -> GraderJudgment: + return GraderJudgment( + judgment_id=f"{run_id}-{grader}", + planned_run_id=run_id, + grader_id=grader, + critical_error_free=value, + atomic_errors=(), + severe_error=False, + rubric_dimensions=RubricDimensions( + constraint_recall=2, + conflict_minority_recognition=2, + unsupported_claim_avoidance=2, + recommendation_correctness=2, + completeness_actionability=2, + readiness_calibration=2, + ), + reviewer_seconds=12.5, + confidence="high", + abstained=False, + rubric_version="rubric-v1", + rubric_hash=DIGEST, + grader_batch="batch-a", + grader_order=1 if grader == "grader-a" else 2, + condition_guess="unknown", + provider_guess="unknown", + ) + + +@pytest.mark.parametrize( + "dimension", + ( + "constraint_recall", + "conflict_minority_recognition", + "unsupported_claim_avoidance", + "recommendation_correctness", + "completeness_actionability", + "readiness_calibration", + ), +) +def test_each_frozen_rubric_dimension_is_bounded_zero_to_two(dimension) -> None: + names = ( + "constraint_recall", + "conflict_minority_recognition", + "unsupported_claim_avoidance", + "recommendation_correctness", + "completeness_actionability", + "readiness_calibration", + ) + values = dict.fromkeys(names, 1) + + assert getattr(RubricDimensions(**{**values, dimension: 0}), dimension) == 0 + assert getattr(RubricDimensions(**{**values, dimension: 2}), dimension) == 2 + with pytest.raises(ValidationError): + RubricDimensions(**{**values, dimension: -1}) + with pytest.raises(ValidationError): + RubricDimensions(**{**values, dimension: 3}) + + +def test_atomic_errors_and_judgment_status_are_typed_and_consistent() -> None: + error = AtomicError( + rubric_item="factuality", + category="unsupported_claim", + severity="severe", + detail="The recommendation contradicts packet p1.", + evidence_packet_ids=("p1",), + ) + judgment = _judgment("run_" + "a" * 24, "grader-a").model_copy( + update={"atomic_errors": (error,), "severe_error": True, "critical_error_free": False} + ) + + assert judgment.atomic_errors[0].severity == "severe" + with pytest.raises(ValidationError, match="severe_error"): + GraderJudgment.model_validate({**judgment.model_dump(mode="json"), "severe_error": False}) + with pytest.raises(ValidationError, match="critical_error_free"): + GraderJudgment.model_validate( + {**judgment.model_dump(mode="json"), "critical_error_free": True} + ) + with pytest.raises(ValidationError, match="abstained"): + GraderJudgment.model_validate( + { + **_judgment("run_" + "b" * 24, "grader-a").model_dump(mode="json"), + "abstained": True, + "critical_error_free": True, + } + ) + + +def test_paid_scoring_requires_two_complete_independent_judgments_per_success() -> None: + manifest, study_run = _paid_study() + successful = [record for record in study_run.records if record.outcome == "success"] + complete = tuple( + _judgment(record.planned_run_id, grader) + for record in successful + for grader in ("grader-a", "grader-b") + ) + + with pytest.raises(ValueError, match="exactly two"): + score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=complete[:-1], + bootstrap_seed=7, + bootstrap_samples=20, + ) + incomplete = complete[0].model_copy(update={"reviewer_seconds": None}) + with pytest.raises(ValueError, match="complete paid-study fields"): + score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=(incomplete, *complete[1:]), + bootstrap_seed=7, + bootstrap_samples=20, + ) + with pytest.raises(ValueError, match="non-success"): + score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=(*complete, _judgment(study_run.records[0].planned_run_id, "grader-a")), + bootstrap_seed=7, + bootstrap_samples=20, + ) + + +def test_paid_reliability_reports_raw_agreement_prevalence_and_strata() -> None: + manifest, study_run = _paid_study() + successful = [record for record in study_run.records if record.outcome == "success"] + judgments = [] + for index, record in enumerate(successful): + judgments.append(_judgment(record.planned_run_id, "grader-a")) + judgments.append(_judgment(record.planned_run_id, "grader-b", value=index != 0)) + + report = score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=tuple(judgments), + bootstrap_seed=7, + bootstrap_samples=20, + ) + + assert report.evidence_classification == "paid_exploratory_pilot" + assert report.reliability.raw_agreement == pytest.approx( + (len(successful) - 1) / len(successful) + ) + assert 0 <= report.reliability.positive_prevalence <= 1 + assert {(item.stratum_type, item.stratum_value) for item in report.reliability.strata} == { + ("family", "operational"), + ("roster", "roster-a"), + ("roster", "roster-b"), + } + assert report.resolved_outcomes[0].resolution == "automatic_non_success" + + +def test_constant_prevalence_kappa_is_undefined_not_perfect() -> None: + assert cohen_kappa([True, True], [True, True]) is None From 523d17f4c139ab50287faace59f1964efcca5bcc Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:18:37 -0400 Subject: [PATCH 04/13] feat(evals): enforce confirmatory analysis gates --- src/conclave/eval_cli.py | 4 + src/conclave/evals/models.py | 34 +++++ src/conclave/evals/runner.py | 8 + src/conclave/evals/scoring.py | 236 ++++++++++++++++++++++++++++- tests/evals/test_analysis_gates.py | 146 ++++++++++++++++++ tests/evals/test_method_grading.py | 13 ++ tests/evals/test_runner.py | 5 + tests/evals/test_study_design.py | 4 + 8 files changed, 448 insertions(+), 2 deletions(-) create mode 100644 tests/evals/test_analysis_gates.py diff --git a/src/conclave/eval_cli.py b/src/conclave/eval_cli.py index 9ecc240..6fff97c 100644 --- a/src/conclave/eval_cli.py +++ b/src/conclave/eval_cli.py @@ -62,11 +62,15 @@ def _validate_run_summary(study_run: StudyRun) -> None: expected_counts = dict(sorted(Counter(record.outcome for record in records).items())) expected_tokens = sum(record.completion_tokens or 0 for record in records) expected_latency = sum(record.latency_ms or 0.0 for record in records) + expected_cost = sum(record.cost_usd for record in records) + expected_deviations = sum(len(record.deviation_codes) for record in records) if ( study_run.total_planned_runs != len(records) or study_run.outcome_counts != expected_counts or study_run.total_completion_tokens != expected_tokens or abs(study_run.total_latency_ms - expected_latency) > 1e-9 + or abs(study_run.total_cost_usd - expected_cost) > 1e-9 + or study_run.total_deviation_count != expected_deviations ): _abort("run artifact summary does not match its immutable records") diff --git a/src/conclave/evals/models.py b/src/conclave/evals/models.py index 21514b1..f3d67b1 100644 --- a/src/conclave/evals/models.py +++ b/src/conclave/evals/models.py @@ -92,6 +92,7 @@ class ExclusionDeviationPolicy(EvalModel): """Predeclared exclusions while keeping execution deviations in analysis.""" predefined_task_exclusions: tuple[str, ...] = () + excluded_task_ids: tuple[str, ...] = () output_level_exclusions_allowed: Literal[False] = False deviations_remain_in_denominator: Literal[True] = True @@ -120,6 +121,28 @@ class BootstrapConfig(EvalModel): unit: Literal["task"] = "task" +class AnalysisGateConfig(EvalModel): + """Canonical DSE-708 confirmatory thresholds frozen before execution.""" + + primary_baseline: ConditionId + alpha: Literal[0.05] = 0.05 + power: Literal[0.80] = 0.80 + minimum_effect: Literal[0.10] = 0.10 + severe_error_noninferiority_margin: Literal[0.02] = 0.02 + readiness_noninferiority_margin: Literal[0.05] = 0.05 + reviewer_effort_max_ratio: Literal[1.20] = 1.20 + p95_latency_max_ratio: Literal[3.0] = 3.0 + absolute_p95_latency_seconds: float = Field(gt=0) + multiplicity_rule: Literal["holm_secondary"] = "holm_secondary" + unit: Literal["task"] = "task" + + @model_validator(mode="after") + def validate_primary_baseline(self) -> AnalysisGateConfig: + if self.primary_baseline == "elite_full": + raise ValueError("primary_baseline must not be elite_full") + return self + + class PriceSnapshot(EvalModel): """Immutable identifier and digest for the price table used by a study.""" @@ -148,6 +171,7 @@ class FrozenStudyDesign(EvalModel): timeout_retry_policy: TimeoutRetryPolicy randomization: RandomizationConfig bootstrap: BootstrapConfig + analysis_gates: AnalysisGateConfig price_snapshot: PriceSnapshot approved_spend_ceiling_usd: float = Field(ge=0) preregistration_id: str | None = None @@ -236,6 +260,8 @@ class ProtocolExecution(EvalModel): completion_tokens: int | None = Field(default=None, ge=0) latency_ms: float | None = Field(default=None, ge=0) error_category: str | None = None + cost_usd: float = Field(default=0.0, ge=0) + deviation_codes: tuple[str, ...] = () class RunRecord(EvalModel): @@ -247,6 +273,8 @@ class RunRecord(EvalModel): completion_tokens: int | None = Field(default=None, ge=0) latency_ms: float | None = Field(default=None, ge=0) error_category: str | None = None + cost_usd: float = Field(default=0.0, ge=0) + deviation_codes: tuple[str, ...] = () class StudyRun(EvalModel): @@ -258,6 +286,8 @@ class StudyRun(EvalModel): outcome_counts: dict[RunOutcome, int] total_completion_tokens: int = Field(ge=0) total_latency_ms: float = Field(ge=0) + total_cost_usd: float = Field(default=0.0, ge=0) + total_deviation_count: int = Field(default=0, ge=0) class ScoreRecord(EvalModel): @@ -302,6 +332,10 @@ def validate_complete_matrix(self) -> StudyManifest: raise ValueError("frozen design hash does not match its contents") if set(self.frozen_design.task_family_map) != set(self.task_ids): raise ValueError("task_family_map must exactly cover manifest task_ids") + if not set(self.frozen_design.exclusion_deviation_policy.excluded_task_ids).issubset( + self.task_ids + ): + raise ValueError("excluded_task_ids must be manifest task IDs") if self.seed != self.frozen_design.randomization.master_seed: raise ValueError("manifest seed must match the frozen randomization master seed") roster_ids = tuple(roster.roster_id for roster in self.frozen_design.rosters) diff --git a/src/conclave/evals/runner.py b/src/conclave/evals/runner.py index 7ca0ff9..a431d3b 100644 --- a/src/conclave/evals/runner.py +++ b/src/conclave/evals/runner.py @@ -106,6 +106,8 @@ async def _execute_cell( completion_tokens=execution.completion_tokens, latency_ms=execution.latency_ms, error_category="output_budget_exceeded", + cost_usd=execution.cost_usd, + deviation_codes=execution.deviation_codes, ) if execution.outcome == "success" and execution.output is None: return RunRecord( @@ -114,6 +116,8 @@ async def _execute_cell( completion_tokens=execution.completion_tokens, latency_ms=execution.latency_ms, error_category="missing_success_output", + cost_usd=execution.cost_usd, + deviation_codes=execution.deviation_codes, ) return RunRecord( planned_run_id=planned_run.planned_run_id, @@ -122,6 +126,8 @@ async def _execute_cell( completion_tokens=execution.completion_tokens, latency_ms=execution.latency_ms, error_category=execution.error_category, + cost_usd=execution.cost_usd, + deviation_codes=execution.deviation_codes, ) except TimeoutError: return RunRecord( @@ -171,4 +177,6 @@ async def run_study( outcome_counts=outcome_counts, total_completion_tokens=sum(record.completion_tokens or 0 for record in records), total_latency_ms=sum(record.latency_ms or 0.0 for record in records), + total_cost_usd=sum(record.cost_usd for record in records), + total_deviation_count=sum(len(record.deviation_codes) for record in records), ) diff --git a/src/conclave/evals/scoring.py b/src/conclave/evals/scoring.py index 462076b..36233eb 100644 --- a/src/conclave/evals/scoring.py +++ b/src/conclave/evals/scoring.py @@ -12,6 +12,7 @@ from .blinding import BlindMap from .models import ( + AnalysisGateConfig, ConditionId, EvalModel, RunOutcome, @@ -65,6 +66,7 @@ class GraderJudgment(EvalModel): grader_order: int | None = Field(default=None, ge=1) condition_guess: ConditionId | Literal["unknown"] | None = None provider_guess: str | None = None + readiness_correct: bool | None = None @model_validator(mode="after") def validate_one_target(self) -> GraderJudgment: @@ -190,6 +192,92 @@ class StudyScoreReport(EvalModel): raw_judgments: tuple[GraderJudgment, ...] adjudications: tuple[AdjudicationRecord, ...] resolved_outcomes: tuple[ResolvedOutcome, ...] + paid_analysis: PaidAnalysisSummary | None = None + + +class PaidAnalysisSummary(EvalModel): + """Task-clustered operational and quality summaries for paid studies.""" + + primary_paired_difference: PairedDifference + severe_error_difference: PairedDifference + readiness_error_difference: PairedDifference + reviewer_effort_ratio: float = Field(gt=0) + p95_latency_ratio: float = Field(gt=0) + elite_p95_latency_seconds: float = Field(ge=0) + total_cost_usd: float = Field(ge=0) + deviation_count: int = Field(ge=0) + excluded_task_ids: tuple[str, ...] + family_directions: dict[str, float] + roster_directions: dict[str, float] + + +class ConfirmatoryGateMetrics(EvalModel): + """Frozen inputs consumed by the deterministic confirmatory gate.""" + + primary_effect: float + primary_lower_95: float + severe_error_upper_95: float + readiness_error_upper_95: float + reviewer_effort_upper_ratio_95: float = Field(gt=0) + p95_latency_ratio: float = Field(gt=0) + elite_p95_latency_seconds: float = Field(ge=0) + family_directions: dict[str, float] = Field(min_length=1) + roster_directions: dict[str, float] = Field(min_length=1) + + +class ConfirmatoryGateResult(EvalModel): + """GO only when every frozen DSE-708 threshold passes.""" + + status: Literal["GO", "REDESIGN"] + failed_gates: tuple[str, ...] + + +def evaluate_confirmatory_gates( + *, + evidence_classification: str, + config: AnalysisGateConfig, + metrics: ConfirmatoryGateMetrics, + expected_frozen_design_hash: str, + observed_frozen_design_hash: str, + expected_preregistration_hash: str, + observed_preregistration_hash: str, +) -> ConfirmatoryGateResult: + """Refuse non-confirmatory/drifted artifacts, then evaluate every canonical gate.""" + + if evidence_classification != "confirmatory": + raise ValueError("confirmatory gate evaluation requires a confirmatory manifest") + if expected_frozen_design_hash != observed_frozen_design_hash: + raise ValueError("observed frozen design hash does not match the preregistered artifact") + if expected_preregistration_hash != observed_preregistration_hash: + raise ValueError("observed preregistration hash does not match the frozen artifact") + failed = [] + checks = ( + ("minimum_effect", metrics.primary_effect >= config.minimum_effect), + ("superiority_interval", metrics.primary_lower_95 > 0), + ( + "severe_error_noninferiority", + metrics.severe_error_upper_95 <= config.severe_error_noninferiority_margin, + ), + ( + "readiness_noninferiority", + metrics.readiness_error_upper_95 <= config.readiness_noninferiority_margin, + ), + ( + "reviewer_effort", + metrics.reviewer_effort_upper_ratio_95 <= config.reviewer_effort_max_ratio, + ), + ("latency_ratio", metrics.p95_latency_ratio <= config.p95_latency_max_ratio), + ( + "latency_ceiling", + metrics.elite_p95_latency_seconds <= config.absolute_p95_latency_seconds, + ), + ("family_direction", all(value > 0 for value in metrics.family_directions.values())), + ("roster_direction", all(value > 0 for value in metrics.roster_directions.values())), + ) + failed.extend(name for name, passed in checks if not passed) + return ConfirmatoryGateResult( + status="GO" if not failed else "REDESIGN", failed_gates=tuple(failed) + ) def wilson_interval(*, successes: int, trials: int, confidence_level: float = 0.95) -> RateInterval: @@ -412,6 +500,7 @@ def _validate_paid_grading( "grader_order", "condition_guess", "provider_guess", + "readiness_correct", ) for run_id, judgments in judgments_by_run.items(): if run_by_id[run_id].outcome != "success": @@ -434,6 +523,17 @@ def _validate_paid_grading( ) +def analysis_planned_runs(manifest: StudyManifest): + """Apply only symmetric, predeclared task exclusions to the analysis matrix.""" + + excluded = ( + set(manifest.frozen_design.exclusion_deviation_policy.excluded_task_ids) + if manifest.frozen_design is not None + else set() + ) + return tuple(run for run in manifest.planned_runs if run.task_id not in excluded) + + def score_study( *, manifest: StudyManifest, @@ -518,12 +618,20 @@ def score_study( ) resolved_by_id = {item.planned_run_id: item for item in resolved} + excluded_task_ids = ( + manifest.frozen_design.exclusion_deviation_policy.excluded_task_ids + if manifest.frozen_design is not None + else () + ) + analysis_runs = analysis_planned_runs(manifest) + if not analysis_runs: + raise ValueError("task exclusions cannot remove every task from analysis") condition_metrics = [] task_condition_values: dict[ConditionId, dict[str, list[bool]]] = defaultdict( lambda: defaultdict(list) ) - for condition_id in dict.fromkeys(run.condition_id for run in manifest.planned_runs): - planned = [run for run in manifest.planned_runs if run.condition_id == condition_id] + for condition_id in dict.fromkeys(run.condition_id for run in analysis_runs): + planned = [run for run in analysis_runs if run.condition_id == condition_id] successes = sum(resolved_by_id[run.planned_run_id].critical_error_free for run in planned) distribution = dict( sorted(Counter(run_by_id[run.planned_run_id].outcome for run in planned).items()) @@ -560,6 +668,129 @@ def score_study( for condition_id in means if condition_id != "elite_full" ) + paid_analysis = None + if manifest.frozen_design is not None: + design = manifest.frozen_design + baseline = design.analysis_gates.primary_baseline + primary = next(item for item in paired if item.baseline_condition_id == baseline) + + def task_means(cell_values: Mapping[str, float]) -> dict[ConditionId, dict[str, float]]: + grouped: dict[ConditionId, dict[str, list[float]]] = defaultdict( + lambda: defaultdict(list) + ) + for run in analysis_runs: + grouped[run.condition_id][run.task_id].append(cell_values[run.planned_run_id]) + return { + condition: {task_id: sum(values) / len(values) for task_id, values in tasks.items()} + for condition, tasks in grouped.items() + } + + severe_cells = { + run.planned_run_id: float( + any(item.severe_error for item in judgments_by_run[run.planned_run_id]) + ) + for run in analysis_runs + } + readiness_error_cells = { + run.planned_run_id: float( + run_by_id[run.planned_run_id].outcome != "success" + or not all( + item.readiness_correct is True for item in judgments_by_run[run.planned_run_id] + ) + ) + for run in analysis_runs + } + effort_cells = { + run.planned_run_id: ( + sum(item.reviewer_seconds or 0 for item in judgments_by_run[run.planned_run_id]) + / len(judgments_by_run[run.planned_run_id]) + if judgments_by_run[run.planned_run_id] + else 0.0 + ) + for run in analysis_runs + } + severe_means = task_means(severe_cells) + readiness_means = task_means(readiness_error_cells) + effort_means = task_means(effort_cells) + frozen_seed = design.bootstrap.seed + frozen_samples = design.bootstrap.samples + severe_difference = paired_bootstrap_difference( + severe_means["elite_full"], + severe_means[baseline], + seed=frozen_seed, + samples=frozen_samples, + baseline_condition_id=baseline, + ) + readiness_difference = paired_bootstrap_difference( + readiness_means["elite_full"], + readiness_means[baseline], + seed=frozen_seed, + samples=frozen_samples, + baseline_condition_id=baseline, + ) + baseline_effort = sum(effort_means[baseline].values()) / len(effort_means[baseline]) + elite_effort = sum(effort_means["elite_full"].values()) / len(effort_means["elite_full"]) + if baseline_effort <= 0: + raise ValueError("paid reviewer-effort baseline must be positive") + latencies = { + condition: sorted( + (run_by_id[run.planned_run_id].latency_ms or 0.0) / 1000 + for run in analysis_runs + if run.condition_id == condition + ) + for condition in (baseline, "elite_full") + } + baseline_p95 = _quantile(latencies[baseline], 0.95) + elite_p95 = _quantile(latencies["elite_full"], 0.95) + if baseline_p95 <= 0: + raise ValueError("paid latency baseline must be positive") + + def directional_effects(attribute: str) -> dict[str, float]: + values = {} + if attribute == "family": + groups = design.task_family_map + names = sorted(set(groups.values())) + + def selected(run, name): + return groups[run.task_id] == name + + else: + names = sorted(roster.roster_id for roster in design.rosters) + + def selected(run, name): + return run.roster_id == name + + for name in names: + elite_values = [ + resolved_by_id[run.planned_run_id].critical_error_free + for run in analysis_runs + if run.condition_id == "elite_full" and selected(run, name) + ] + baseline_values = [ + resolved_by_id[run.planned_run_id].critical_error_free + for run in analysis_runs + if run.condition_id == baseline and selected(run, name) + ] + values[name] = sum(elite_values) / len(elite_values) - sum(baseline_values) / len( + baseline_values + ) + return values + + paid_analysis = PaidAnalysisSummary( + primary_paired_difference=primary, + severe_error_difference=severe_difference, + readiness_error_difference=readiness_difference, + reviewer_effort_ratio=elite_effort / baseline_effort, + p95_latency_ratio=elite_p95 / baseline_p95, + elite_p95_latency_seconds=elite_p95, + total_cost_usd=sum(run_by_id[run.planned_run_id].cost_usd for run in analysis_runs), + deviation_count=sum( + len(run_by_id[run.planned_run_id].deviation_codes) for run in analysis_runs + ), + excluded_task_ids=tuple(sorted(excluded_task_ids)), + family_directions=directional_effects("family"), + roster_directions=directional_effects("roster"), + ) return StudyScoreReport( study_id=manifest.study_id, evidence_classification=manifest.evidence_classification, @@ -573,4 +804,5 @@ def score_study( raw_judgments=tuple(raw_judgments), adjudications=tuple(adjudications), resolved_outcomes=tuple(resolved), + paid_analysis=paid_analysis, ) diff --git a/tests/evals/test_analysis_gates.py b/tests/evals/test_analysis_gates.py new file mode 100644 index 0000000..06016d1 --- /dev/null +++ b/tests/evals/test_analysis_gates.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from conclave.evals.models import AnalysisGateConfig +from conclave.evals.protocols import CONDITION_IDS, build_study_manifest +from conclave.evals.scoring import ( + ConfirmatoryGateMetrics, + analysis_planned_runs, + evaluate_confirmatory_gates, +) +from tests.evals.test_study_design import _design, _tasks + + +def _config(**updates): + values = { + "primary_baseline": "self_refine", + "absolute_p95_latency_seconds": 180, + } + values.update(updates) + return AnalysisGateConfig(**values) + + +def test_analysis_gate_config_freezes_canonical_dse_708_thresholds() -> None: + config = _config() + + assert config.alpha == 0.05 + assert config.power == 0.80 + assert config.minimum_effect == 0.10 + assert config.severe_error_noninferiority_margin == 0.02 + assert config.readiness_noninferiority_margin == 0.05 + assert config.reviewer_effort_max_ratio == 1.20 + assert config.p95_latency_max_ratio == 3.0 + assert config.multiplicity_rule == "holm_secondary" + assert config.unit == "task" + with pytest.raises(ValidationError, match="primary_baseline"): + _config(primary_baseline="elite_full") + + +def _passing_metrics(**updates): + values = { + "primary_effect": 0.12, + "primary_lower_95": 0.01, + "severe_error_upper_95": 0.02, + "readiness_error_upper_95": 0.05, + "reviewer_effort_upper_ratio_95": 1.20, + "p95_latency_ratio": 3.0, + "elite_p95_latency_seconds": 180, + "family_directions": {"operational": 0.01, "stewardship": 0.02}, + "roster_directions": {"roster-a": 0.01, "roster-b": 0.01}, + } + values.update(updates) + return ConfirmatoryGateMetrics(**values) + + +def test_confirmatory_gate_boundaries_are_inclusive_and_every_gate_is_required() -> None: + config = _config() + result = evaluate_confirmatory_gates( + evidence_classification="confirmatory", + config=config, + metrics=_passing_metrics(), + expected_frozen_design_hash="sha256:" + "a" * 64, + observed_frozen_design_hash="sha256:" + "a" * 64, + expected_preregistration_hash="sha256:" + "b" * 64, + observed_preregistration_hash="sha256:" + "b" * 64, + ) + + assert result.status == "GO" + for field, value in ( + ("primary_effect", 0.099), + ("primary_lower_95", 0.0), + ("severe_error_upper_95", 0.021), + ("readiness_error_upper_95", 0.051), + ("reviewer_effort_upper_ratio_95", 1.201), + ("p95_latency_ratio", 3.001), + ("elite_p95_latency_seconds", 180.1), + ): + failed = evaluate_confirmatory_gates( + evidence_classification="confirmatory", + config=config, + metrics=_passing_metrics(**{field: value}), + expected_frozen_design_hash="sha256:" + "a" * 64, + observed_frozen_design_hash="sha256:" + "a" * 64, + expected_preregistration_hash="sha256:" + "b" * 64, + observed_preregistration_hash="sha256:" + "b" * 64, + ) + assert failed.status == "REDESIGN" + + failed_direction = evaluate_confirmatory_gates( + evidence_classification="confirmatory", + config=config, + metrics=_passing_metrics(family_directions={"operational": 0.0}), + expected_frozen_design_hash="sha256:" + "a" * 64, + observed_frozen_design_hash="sha256:" + "a" * 64, + expected_preregistration_hash="sha256:" + "b" * 64, + observed_preregistration_hash="sha256:" + "b" * 64, + ) + assert failed_direction.status == "REDESIGN" + + +def test_confirmatory_gate_refuses_exploratory_or_hash_drift() -> None: + kwargs = dict( + config=_config(), + metrics=_passing_metrics(), + expected_frozen_design_hash="sha256:" + "a" * 64, + observed_frozen_design_hash="sha256:" + "a" * 64, + expected_preregistration_hash="sha256:" + "b" * 64, + observed_preregistration_hash="sha256:" + "b" * 64, + ) + with pytest.raises(ValueError, match="confirmatory"): + evaluate_confirmatory_gates(evidence_classification="paid_exploratory_pilot", **kwargs) + with pytest.raises(ValueError, match="frozen design hash"): + evaluate_confirmatory_gates( + evidence_classification="confirmatory", + **{**kwargs, "observed_frozen_design_hash": "sha256:" + "c" * 64}, + ) + with pytest.raises(ValueError, match="preregistration hash"): + evaluate_confirmatory_gates( + evidence_classification="confirmatory", + **{**kwargs, "observed_preregistration_hash": "sha256:" + "c" * 64}, + ) + + +def test_task_exclusion_is_symmetric_across_every_roster_condition_cell() -> None: + design = _design().model_copy( + update={ + "exclusion_deviation_policy": _design().exclusion_deviation_policy.model_copy( + update={"excluded_task_ids": ("task-a",)} + ) + } + ) + manifest = build_study_manifest( + study_id="excluded-task", + tasks=_tasks(), + replicates=1, + seed=20260717, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 100), + frozen_design=design, + ) + + included = analysis_planned_runs(manifest) + assert {run.task_id for run in included} == {"task-b"} + assert len(included) == 2 * 6 + assert {run.roster_id for run in included} == {"roster-a", "roster-b"} + assert {run.condition_id for run in included} == set(CONDITION_IDS) diff --git a/tests/evals/test_method_grading.py b/tests/evals/test_method_grading.py index ebeba5c..c7e1dcf 100644 --- a/tests/evals/test_method_grading.py +++ b/tests/evals/test_method_grading.py @@ -4,6 +4,7 @@ from pydantic import ValidationError from conclave.evals.models import ( + AnalysisGateConfig, BootstrapConfig, ExclusionDeviationPolicy, FrozenStudyDesign, @@ -59,6 +60,9 @@ def _paid_study(): timeout_retry_policy=TimeoutRetryPolicy(timeout_seconds=60, retry_attempts=0), randomization=RandomizationConfig(master_seed=17), bootstrap=BootstrapConfig(seed=19, samples=100), + analysis_gates=AnalysisGateConfig( + primary_baseline="self_refine", absolute_p95_latency_seconds=180 + ), price_snapshot=PriceSnapshot( snapshot_id="prices-v1", captured_at="2026-07-17T00:00:00Z", @@ -81,6 +85,9 @@ def _paid_study(): outcome="failed" if index == 0 else "success", output=None if index == 0 else "answer", error_category="executor_error" if index == 0 else None, + latency_ms=100.0 + index, + cost_usd=0.01, + deviation_codes=("retry_used",) if index == 1 else (), ) for index, planned in enumerate(manifest.planned_runs) ) @@ -91,6 +98,8 @@ def _paid_study(): outcome_counts={"failed": 1, "success": len(records) - 1}, total_completion_tokens=0, total_latency_ms=0, + total_cost_usd=0.12, + total_deviation_count=1, ) return manifest, study_run @@ -120,6 +129,7 @@ def _judgment(run_id: str, grader: str, *, value: bool = True) -> GraderJudgment grader_order=1 if grader == "grader-a" else 2, condition_guess="unknown", provider_guess="unknown", + readiness_correct=True, ) @@ -245,6 +255,9 @@ def test_paid_reliability_reports_raw_agreement_prevalence_and_strata() -> None: ("roster", "roster-b"), } assert report.resolved_outcomes[0].resolution == "automatic_non_success" + assert report.paid_analysis.primary_paired_difference.baseline_condition_id == "self_refine" + assert report.paid_analysis.primary_paired_difference.task_count == 1 + assert report.paid_analysis.excluded_task_ids == () def test_constant_prevalence_kappa_is_undefined_not_perfect() -> None: diff --git a/tests/evals/test_runner.py b/tests/evals/test_runner.py index b57f59e..f039581 100644 --- a/tests/evals/test_runner.py +++ b/tests/evals/test_runner.py @@ -37,6 +37,8 @@ async def executor(*, task, planned_run, max_output_tokens): output=f"{task.task_id}:{planned_run.condition_id}", completion_tokens=11, latency_ms=2.5, + cost_usd=0.01, + deviation_codes=("retry_used",), ) study_run = await run_study( @@ -55,6 +57,9 @@ async def executor(*, task, planned_run, max_output_tokens): assert study_run.outcome_counts == {"success": 24} assert study_run.total_completion_tokens == 264 assert study_run.total_latency_ms == 60.0 + assert study_run.total_cost_usd == pytest.approx(0.24) + assert study_run.total_deviation_count == 24 + assert study_run.records[0].deviation_codes == ("retry_used",) with pytest.raises(ValidationError): study_run.records[0].output = "changed" diff --git a/tests/evals/test_study_design.py b/tests/evals/test_study_design.py index 6c75dc9..89c7785 100644 --- a/tests/evals/test_study_design.py +++ b/tests/evals/test_study_design.py @@ -6,6 +6,7 @@ from pydantic import ValidationError from conclave.evals.models import ( + AnalysisGateConfig, BootstrapConfig, ExclusionDeviationPolicy, FrozenStudyDesign, @@ -74,6 +75,9 @@ def _design( timeout_retry_policy=TimeoutRetryPolicy(timeout_seconds=120, retry_attempts=1), randomization=RandomizationConfig(master_seed=20260717), bootstrap=BootstrapConfig(seed=991, samples=10000), + analysis_gates=AnalysisGateConfig( + primary_baseline="self_refine", absolute_p95_latency_seconds=180 + ), price_snapshot=PriceSnapshot( snapshot_id="prices-2026-07-17", captured_at="2026-07-17T12:00:00Z", From e69d99ff05ff8873348e3f16706c1898f710451e Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:19:57 -0400 Subject: [PATCH 05/13] fix(evals): label paid and confirmatory reports honestly --- src/conclave/evals/reporting.py | 12 ++++++++++-- tests/evals/test_reporting.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/conclave/evals/reporting.py b/src/conclave/evals/reporting.py index f3ad1e0..ffab784 100644 --- a/src/conclave/evals/reporting.py +++ b/src/conclave/evals/reporting.py @@ -1,4 +1,4 @@ -"""Deterministic JSON and Markdown rendering for exploratory eval reports.""" +"""Deterministic JSON and Markdown rendering for evaluation reports.""" from __future__ import annotations @@ -12,8 +12,16 @@ def render_markdown_report(report: StudyScoreReport) -> str: """Render a compact human report without promoting exploratory evidence.""" + evidence_labels = { + "synthetic_exploratory": "SYNTHETIC / EXPLORATORY", + "paid_exploratory_pilot": "PAID / EXPLORATORY PILOT", + "confirmatory": "CONFIRMATORY / GATE EVALUATION REQUIRED", + } + evidence_label = evidence_labels.get( + report.evidence_classification, report.evidence_classification.upper() + ) lines = [ - "# Conclave Elite Evaluation — SYNTHETIC / EXPLORATORY", + f"# Conclave Elite Evaluation — {evidence_label}", "", f"Study: `{report.study_id}`", "", diff --git a/tests/evals/test_reporting.py b/tests/evals/test_reporting.py index c548fd2..4169b74 100644 --- a/tests/evals/test_reporting.py +++ b/tests/evals/test_reporting.py @@ -64,6 +64,17 @@ def test_markdown_report_is_explicitly_exploratory_and_complete() -> None: assert "confirmatory" in markdown.lower() +def test_markdown_report_uses_the_recorded_evidence_classification() -> None: + report = _synthetic_report().model_copy( + update={"evidence_classification": "paid_exploratory_pilot"} + ) + + markdown = render_markdown_report(report) + + assert "PAID / EXPLORATORY PILOT" in markdown + assert "SYNTHETIC / EXPLORATORY" not in markdown + + def test_report_bundle_writes_machine_json_and_human_markdown(tmp_path) -> None: report = _synthetic_report() json_path = tmp_path / "nested" / "report.json" From acedc8fa3e1e6a689ff2bbdb095d2f92b137875b Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:27:17 -0400 Subject: [PATCH 06/13] feat(evals): add frozen 24-task elite pilot pack --- studies/elite_pilot_v1/README.md | 38 +++++ .../confirmatory_preregistration.md | 68 ++++++++ studies/elite_pilot_v1/grader_keys.json | 149 ++++++++++++++++++ studies/elite_pilot_v1/pilot_protocol.md | 80 ++++++++++ studies/elite_pilot_v1/public_tasks.json | 149 ++++++++++++++++++ tests/evals/test_pilot_pack.py | 102 ++++++++++++ 6 files changed, 586 insertions(+) create mode 100644 studies/elite_pilot_v1/README.md create mode 100644 studies/elite_pilot_v1/confirmatory_preregistration.md create mode 100644 studies/elite_pilot_v1/grader_keys.json create mode 100644 studies/elite_pilot_v1/pilot_protocol.md create mode 100644 studies/elite_pilot_v1/public_tasks.json create mode 100644 tests/evals/test_pilot_pack.py diff --git a/studies/elite_pilot_v1/README.md b/studies/elite_pilot_v1/README.md new file mode 100644 index 0000000..075d180 --- /dev/null +++ b/studies/elite_pilot_v1/README.md @@ -0,0 +1,38 @@ +# Elite Pilot v1 + +This directory is a **synthetic exploratory** evaluation pack for debugging the Conclave H1 +study method. It is not a production benchmark, a confirmatory study, or evidence that Elite +improves decisions. Its results **must not support product-quality claims**. + +## Contents + +- `public_tasks.json` contains the 24 synthetic tasks visible to every experimental condition. +- `grader_keys.json` contains expected decisions, critical errors, and rubric anchors. It is + grader-only and must never be loaded by a protocol runner or included in a model prompt. +- `pilot_protocol.md` freezes the exploratory execution and grading rules. +- `confirmatory_preregistration.md` is a gated template for a later, independently authored + held-out study. It is not an active preregistration. + +The pack has two prespecified macro-families with 12 tasks each, six subfamilies with four +tasks each, and balanced tiers and expected readiness outcomes: + +| Axis | Distribution | +| --- | --- | +| Macro-family | 12 `operational_execution`; 12 `organizational_stewardship` | +| Subfamily | 4 each: procurement, release operations, product experiments, capital allocation, governance, staffing | +| Difficulty | 8 each: `tier_1`, `tier_2`, `tier_3` | +| Keyed readiness | 8 each: `ready`, `not_ready`, `indeterminate` | + +All facts, organizations, people, products, and prices are fictional. The tasks require no +current knowledge and prohibit external assumptions. Public packets and grader keys use the +immutable `conclave_eval_v1` contracts and can be loaded with +`conclave.evals.dataset.load_public_tasks` and `load_grader_keys`. + +## Frozen pilot-only boundary + +Pilot v1 may debug task clarity, rubric wording, grader agreement, execution reliability, and +confirmatory sample-size assumptions. It may not establish superiority, authorize product +claims, tune a confirmatory holdout, or influence real decisions. Any material task, key, +prompt, rubric, or protocol change creates a new version and new hashes; it must not silently +replace this pack. + diff --git a/studies/elite_pilot_v1/confirmatory_preregistration.md b/studies/elite_pilot_v1/confirmatory_preregistration.md new file mode 100644 index 0000000..af6b395 --- /dev/null +++ b/studies/elite_pilot_v1/confirmatory_preregistration.md @@ -0,0 +1,68 @@ +# Confirmatory Study Preregistration Gate + +> **Status:** This is **not a completed preregistration** and does not authorize a +> confirmatory run. Every bracketed field must be resolved, independently reviewed, signed, +> hashed, and frozen before holdout unsealing or provider execution. + +## Required declarations + +- Study ID and immutable artifact location: `[required]` +- Owners for methods, corpus, execution, grading, and adjudication: `[required]` +- Primary hypothesis and direction: `[required]` +- Primary endpoint: failure-inclusive critical-error-free decision rate: `[confirm]` +- Primary comparison and multiplicity treatment: `[required]` +- Minimum practically important effect: `[required]` +- Power, alpha, sample-size method, and assumptions justified from pilot variance: `[required]` +- Two prespecified macro-families and minimum family-specific sample sizes: `[required]` +- Conditions, prompts, roster, provider/model versions, budgets, replicates, and seed: `[required]` +- Inclusion, failure, retry, exclusion, stopping, and missing-data rules: `[required]` +- Secondary dimensions, calibration analysis, grader-time, cost, and latency guardrails: `[required]` +- Go, redesign, and kill thresholds: `[required]` + +## Independent holdout construction + +The confirmatory corpus must be authored as a separate project by people who cannot see pilot +outputs while authoring keys. It must use **new scenario archetypes**, entities, numbers, +packet prose, constraint interactions, and minority-view traps. Simple **parameter swaps** or +paraphrases of pilot tasks are prohibited. Pilot task or answer text may not enter examples, +few-shot context, prompt tuning, grader training, or rubric demonstrations. + +Before acceptance, run exact-ID and exact-text checks, an **eight-token** contiguous-overlap +screen, MinHash or equivalent near-duplicate detection, embedding similarity review, and a +manual semantic-leakage review. Search repository code, tests, documentation, replay fixtures, +model-facing prompts, and prior reports. Replace a leaked or semantically duplicated holdout +task; do not repair it by editing only its answer key. + +Public tasks and grader keys remain separate. The execution identity cannot read the keys. +Corpus authors do not grade outputs they authored unless declared and sensitivity-tested. +Grader training uses separate calibration examples that are in neither pilot nor holdout. + +## Freeze and unsealing gate + +Before **unsealing** any confirmatory task to a model, freeze and publish internally the +cryptographic hashes for public tasks, grader keys, prompt, conditions, roster, model/provider +versions, generation settings, token and dollar ceilings, replicate count, seed, blind-map +procedure, rubric, critical errors, grader training set, exclusions, retry policy, analysis +code, primary comparison, minimum effect, sample size, stopping rules, and decision gates. + +An independent reviewer must attest that: + +1. the pilot and holdout are semantically distinct; +2. no model, protocol author, or runner has received grader-only material; +3. budgets and evidence access are condition-matched; +4. the full failure-inclusive analysis can run from frozen artifacts; +5. provider spend and automatic-stop authority are approved; and +6. H0 and the H1 harness revision are merged, pinned, and green. + +If any item fails, the confirmatory study remains sealed. After unsealing, no prompt, task, +key, rubric, roster, budget, outcome rule, or analysis change is permitted. Deviations are +logged and the primary result remains based on the preregistered analysis. + +## Reporting boundary + +Report task-paired effects and uncertainty, family distributions, severe-error and calibration +guardrails, execution failures, tokens, dollar cost, latency, reviewer minutes, raw grader +agreement, and adjudication. A go decision requires the preregistered improvement and no +material regression on severe errors, readiness calibration, reviewer effort, cost, or +latency. Results that miss the gate trigger redesign or kill; they must not be reframed as +proof through post-hoc slices. diff --git a/studies/elite_pilot_v1/grader_keys.json b/studies/elite_pilot_v1/grader_keys.json new file mode 100644 index 0000000..05b3137 --- /dev/null +++ b/studies/elite_pilot_v1/grader_keys.json @@ -0,0 +1,149 @@ +{ + "schema_version": "conclave_eval_v1", + "grader_keys": [ + { + "task_id": "PRC-01", + "required_facts": ["expected_readiness:ready", "recommendation:select Courier A", "A alone meets cost, on-time, and certificate constraints", "the premium-quality minority claim is unmeasured", "next_action:contract owner verifies certificate and monitors on-time rate"], + "critical_errors": ["select Courier B or C", "claim Courier A has proven superior quality", "declare not_ready or indeterminate despite complete qualifying evidence"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "PRC-02", + "required_facts": ["expected_readiness:not_ready", "recommendation:select no vendor", "A fails audit, B fails deletion, and C fails price", "verbal or future terms are not current evidence", "next_action:procurement obtains a compliant signed offer or approved constraint change"], + "critical_errors": ["select A, B, or C now", "count a verbal discount or future amendment as satisfied", "omit a known hard-constraint violation"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "PRC-03", + "required_facts": ["expected_readiness:indeterminate", "recommendation:do not select A or B yet", "both meet known power and cost constraints", "boundary noise cannot be derived from the supplied catalog values", "next_action:facilities owner commissions a site-boundary acoustic assessment"], + "critical_errors": ["declare A or B ready", "infer boundary dBA without a supplied method", "treat the minority assertion as measurement"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "PRC-04", + "required_facts": ["expected_readiness:indeterminate", "recommendation:do not select a service yet", "C fails p95 recovery", "A lacks p95 and subcontractor evidence while B lacks subcontractor evidence", "next_action:recovery owner obtains A p95 and lists plus B subcontractor documentation"], + "critical_errors": ["select A or B as ready", "substitute A median for p95", "ignore undocumented critical subcontractors"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "REL-01", + "required_facts": ["expected_readiness:ready", "recommendation:release RC1", "RC1 meets all three release gates", "RC2 fails rollback and the 3% latency note is not a defined gate", "next_action:release owner deploys RC1 with latency monitoring and rollback trigger"], + "critical_errors": ["release RC2", "block RC1 solely on the unspecified latency threshold", "claim RC1 has no performance concern"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "REL-02", + "required_facts": ["expected_readiness:not_ready", "recommendation:do not deploy now", "the 30-hour backup violates the 24-hour gate", "rollback and incident gates pass but do not override the backup failure", "next_action:operations takes and verifies a new backup then rechecks every gate"], + "critical_errors": ["deploy now", "treat probably unchanged as evidence", "call the known blocker merely indeterminate"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "REL-03", + "required_facts": ["expected_readiness:indeterminate", "recommendation:choose none of restart, rollback, or failover yet", "telemetry conflicts and chronology is unresolved", "rollback integrity and failover lag are unknown", "next_action:incident commander caps traffic, preserves logs, synchronizes the timeline, measures lag, and verifies the artifact"], + "critical_errors": ["assert memory leak or database saturation as established cause", "execute rollback or failover without its safety evidence", "offer no immediate containment or evidence-preservation action"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "REL-04", + "required_facts": ["expected_readiness:not_ready", "recommendation:do not go live", "security and user-acceptance gates pass", "two critical accessibility failures violate the zero-failure gate", "next_action:accessibility owner fixes and retests keyboard navigation"], + "critical_errors": ["go live", "downgrade the accessibility gate because of a workaround", "omit the known critical failures"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "PRD-01", + "required_facts": ["expected_readiness:ready", "recommendation:launch Plan A", "A alone meets every experiment gate", "B lacks a holdout and C collects a prohibited field", "next_action:experiment owner preserves preregistration and monitors guardrails"], + "critical_errors": ["launch Plan B or C", "treat greater exposure as a substitute for a holdout", "invent expected experiment performance"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "PRD-02", + "required_facts": ["expected_readiness:not_ready", "recommendation:do not roll out pricing", "the churn interval permits harm above the one-point gate", "severe billing tickets independently violate the support gate", "next_action:product owner resolves confusion and runs evidence able to bound churn harm"], + "critical_errors": ["roll out pricing", "claim the churn gate passed from the point estimate", "ignore the severe support signal because revenue rose"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "PRD-03", + "required_facts": ["expected_readiness:indeterminate", "recommendation:do not approve the sunset yet", "usage and savings gates pass", "active contractual dependency status is missing", "next_action:contract owner completes and signs the dependency inventory"], + "critical_errors": ["approve the sunset", "treat a reviewer's memory as the contract record", "declare not_ready as though a dependency were proven"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "PRD-04", + "required_facts": ["expected_readiness:indeterminate", "recommendation:select neither region yet", "Region A passes known margin and coverage but lacks clearance", "Region B fails the known margin gate", "next_action:legal and tax owners provide written Region A clearance"], + "critical_errors": ["select Region A as ready", "select Region B despite margin failure", "invent Region A regulatory status"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "CAP-01", + "required_facts": ["expected_readiness:ready", "recommendation:select Project B", "B alone meets cost, schedule, and security constraints", "A's hypothetical patch is not evidence and C exceeds budget", "next_action:project owner contracts B with schedule and security checkpoints"], + "critical_errors": ["select A or C", "count the hypothetical patch as a demonstrated score", "declare no option qualifies"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "CAP-02", + "required_facts": ["expected_readiness:not_ready", "recommendation:select no investment", "A fails operating cost, B fails capital cost, and C fails payback", "the B discount is unsigned and the operating-cost constraint remains binding", "next_action:finance obtains compliant terms or authorized constraint changes"], + "critical_errors": ["select A, B, or C", "count the unsigned discount", "ignore a stated financial constraint"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "CAP-03", + "required_facts": ["expected_readiness:indeterminate", "recommendation:select neither migration yet", "both pass known cost and downtime", "neither has independently demonstrated recovery risk", "next_action:recovery owner runs an independent rehearsal for A and B"], + "critical_errors": ["select A as ready", "call a vendor estimate independently demonstrated", "invent B's failure probability"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "CAP-04", + "required_facts": ["expected_readiness:ready", "recommendation:select Plan B", "B meets exact total and every minimum allocation", "A lacks documentation and C lacks patching", "next_action:engineering owner assigns the B allocations and preserves the one-week reserve"], + "critical_errors": ["select Plan A or C", "move reserve to unapproved feature work", "misstate the total engineer-weeks"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "GOV-01", + "required_facts": ["expected_readiness:ready", "recommendation:select Policy A", "A alone meets raw retention, aggregate retention, and verification", "the investigation preference has no approved exception", "next_action:policy owner implements A and audits deletion verification"], + "critical_errors": ["select B or C", "waive the raw-log limit", "invent an investigation exception"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "GOV-02", + "required_facts": ["expected_readiness:not_ready", "recommendation:do not deploy the automated tool", "automated declines and absent appeal violate two controls", "audit logs and accuracy do not cure those violations", "next_action:governance owner redesigns, implements appeal, and retests before use"], + "critical_errors": ["deploy the tool", "equate accuracy with governance compliance", "treat a roadmap promise as an implemented control"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "GOV-03", + "required_facts": ["expected_readiness:indeterminate", "recommendation:do not transfer or make a final approval decision", "the clauses conflict and precedence is absent", "both stakeholder positions lack authoritative interpretation", "next_action:authorized contract owner determines precedence in writing"], + "critical_errors": ["approve or deny as legally settled", "invent clause precedence", "transfer data while authorization is unresolved"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "GOV-04", + "required_facts": ["expected_readiness:ready", "recommendation:select Policy B", "B alone meets least privilege, separation, and logged one-hour break-glass", "A violates least privilege and C violates separation and logging", "next_action:access owner implements B and schedules periodic review"], + "critical_errors": ["select A or C", "prioritize convenience over a hard control", "omit C's unlogged emergency access"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "STF-01", + "required_facts": ["expected_readiness:ready", "recommendation:select Plan A", "A meets composition, junior, and cost constraints", "B lacks a second operator and C exceeds cost", "next_action:staffing owner confirms the A roster and coverage"], + "critical_errors": ["select B or C", "treat lower cost as overriding required composition", "invent qualifications for the junior"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "STF-02", + "required_facts": ["expected_readiness:not_ready", "recommendation:hire none of A, B, or C now", "A fails certification, B fails start date, and C fails salary", "no exception or negotiated offer exists", "next_action:hiring owner finds a compliant candidate or obtains an explicit approved exception"], + "critical_errors": ["select A, B, or C now", "assume C will negotiate", "ignore the absence of exception authority"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "STF-03", + "required_facts": ["expected_readiness:indeterminate", "recommendation:do not outsource yet", "demand forecast is missing", "five samples cannot establish the true defect rate at the required confidence", "next_action:operations owner obtains an approved forecast and adequately powered partner-quality sample"], + "critical_errors": ["approve outsourcing", "claim the partner's true defect rate is zero", "ignore the missing demand forecast"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + }, + { + "task_id": "STF-04", + "required_facts": ["expected_readiness:not_ready", "recommendation:select none of Plans A, B, or C", "A and C violate coverage while B violates deadline", "the proposed 30-minute exception is not approved", "next_action:training owner creates a compliant staggered plan before Friday"], + "critical_errors": ["select A, B, or C", "treat the deadline or coverage minimum as optional", "claim a coverage exception exists"], + "rubric": {"constraint_recall": "2=all hard constraints explicit and cited; 1=one noncritical omission; 0=material constraint missed or misstated", "conflict_minority_recognition": "2=conflict and minority view accurately represented; 1=mentioned incompletely; 0=ignored or treated as verified", "unsupported_claim_avoidance": "2=no invented material fact and inferences labeled; 1=one immaterial uncited assertion; 0=invented decision-driving fact", "recommendation_correctness": "2=matches keyed recommendation with sound rationale; 1=directionally correct but imprecise; 0=wrong course", "completeness_actionability": "2=decision, checks, conflict, owner, action, trigger, and unknowns; 1=one noncritical element absent; 0=not executable", "readiness_calibration": "2=matches keyed readiness and explains why; 1=right direction with weak basis; 0=wrong readiness"} + } + ] +} diff --git a/studies/elite_pilot_v1/pilot_protocol.md b/studies/elite_pilot_v1/pilot_protocol.md new file mode 100644 index 0000000..982cdec --- /dev/null +++ b/studies/elite_pilot_v1/pilot_protocol.md @@ -0,0 +1,80 @@ +# Elite Pilot v1 Protocol + +## Purpose and decision boundary + +This is a frozen, synthetic, exploratory pilot. It tests whether the H1 harness, task design, +atomic rubric, blinding, and adjudication process are usable. It cannot demonstrate that a +condition is better, justify a product-quality claim, or change a real decision. The primary +pilot outcome is method readiness for a separately authored confirmatory study. + +## Frozen corpus and prompt + +The study uses all 24 tasks in `public_tasks.json`: 12 per macro-family, four per subfamily, +eight per difficulty tier, and eight per keyed readiness class. Every condition receives the +same task, packet, prompt, and declared output-token ceiling. The prompt is: + +> Using only the reference packet, choose the best course of action. Return: (1) +> Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, +> (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do +> not invent facts. + +The six frozen comparison conditions are `single_frontier`, `self_refine`, +`independent_synthesis`, `critique_only`, `revision_only`, and `elite_full`. The manifest +predeclares the complete task-by-condition-by-replicate matrix and a seeded condition order. +Planned output-token ceilings must satisfy the harness's budget-match tolerance. The public +task hash, prompt, conditions, model and provider identities, generation settings, budgets, +replicate count, seed, software revision, and replay-artifact hashes are recorded before run. + +## Execution + +Only `public_tasks.json` may enter execution. `grader_keys.json` is grader-only and must be +kept outside the runner's readable inputs. Every planned cell produces exactly one record. +Timeouts, malformed output, abstentions, incomplete output, provider errors, and missing +records are non-successes; **failures remain in the denominator**. No cell may be rerun or +excluded because its answer is inconvenient. Any permitted infrastructure retry must be +predeclared and retain every attempt in the audit artifact. + +This repository pack does not authorize paid calls. A live pilot requires a separately +approved spend ceiling, an automatic stop below that ceiling, pinned model identifiers, and a +fresh manifest. Offline replay is the default until those gates are satisfied. + +## Atomic grading + +Outputs are randomized behind opaque identifiers and presented without condition, provider, +model, timing, token, or execution-order labels. Views use a predeclared length policy; graders +may not see grader keys until model execution is complete. Two graders independently score +each successful output before discussion. They assign 0, 1, or 2 on exactly six dimensions: + +1. `constraint_recall` +2. `conflict_minority_recognition` +3. `unsupported_claim_avoidance` +4. `recommendation_correctness` +5. `completeness_actionability` +6. `readiness_calibration` + +For each dimension, 2 means all keyed elements are accurate and supported; 1 means the +direction is correct with one noncritical omission; 0 means a wrong, unsupported, or material +omission. Unsupported-claim avoidance is 2 only when no material fact is invented and +inferences are labeled. Completeness/actionability is 2 only when the response supplies the +decision, constraint check, conflict, concrete action, owner, trigger, and unknowns. + +The primary metric is the failure-inclusive **critical-error-free decision rate**. Any keyed +critical error forces `critical_error_free=false` regardless of dimension totals. Secondary +outputs are the six separate dimension distributions, severe and unsupported error counts, +readiness confusion matrix, conflict/minority recall, tokens, latency, failures, and grader +minutes. No composite quality score is reported. + +Grader disagreements retain both raw judgments and receive a separate adjudication citing +exactly the source judgments. Report Cohen's kappa, disagreement count, adjudication rate, +Wilson intervals for rates, and seeded task-paired bootstrap intervals for Elite-minus-each- +baseline differences. Report macro-family and subfamily results descriptively; this pilot is +not powered for confirmatory subgroup claims. + +## Frozen exclusions and changes + +Exclude no task or output after execution. Pre-execution rejection is allowed only for a +schema-invalid or hash-mismatched entire pack, which aborts the study rather than deleting +cells. Changes to a scenario, packet, key, prompt, condition, dimension, critical-error rule, +budget rule, or analysis create a new pack version. Pilot observations may improve a future +rubric and estimate variance; they may not alter, reveal, or train on confirmatory tasks. + diff --git a/studies/elite_pilot_v1/public_tasks.json b/studies/elite_pilot_v1/public_tasks.json new file mode 100644 index 0000000..ddde3c9 --- /dev/null +++ b/studies/elite_pilot_v1/public_tasks.json @@ -0,0 +1,149 @@ +{ + "schema_version": "conclave_eval_v1", + "tasks": [ + { + "task_id": "PRC-01", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Courier hard constraints: cost at most $14 per order, on-time rate at least 95%, and a current cold-chain certificate.", "[P2] Courier A: $12, 96%, current certificate. Courier B: $10, 92%, current certificate. Courier C: $15, 98%, current certificate.", "[P3] A recent Courier A sample delivered 48 of 50 orders on time (96%).", "[P4] A minority reviewer prefers Courier C for premium quality; the packet supplies no quality measure."], + "metadata": {"macro_family": "operational_execution", "subfamily": "procurement", "tier": "tier_1"} + }, + { + "task_id": "PRC-02", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Software hard constraints: annual price at most $60,000, deletion within 30 days, and a customer audit right.", "[P2] A: $55,000, 30-day deletion, no audit right. B: $58,000, 90-day deletion, audit right. C: $64,000, 30-day deletion, audit right.", "[P3] C's salesperson verbally expects a discount, but no signed quote exists.", "[P4] A minority reviewer says the audit right can be added after signing; no amendment is supplied."], + "metadata": {"macro_family": "operational_execution", "subfamily": "procurement", "tier": "tier_2"} + }, + { + "task_id": "PRC-03", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Generator hard constraints: no more than 65 dBA at the site boundary, at least 300 kW, and price at most $90,000.", "[P2] A: 320 kW, $84,000, catalog rating 61 dBA at 7 m. B: 305 kW, $79,000, catalog rating 64 dBA at 7 m.", "[P3] The boundary is 28 m away, but terrain and barriers are unmeasured and no propagation method is provided.", "[P4] A minority reviewer says the catalog values prove boundary compliance."], + "metadata": {"macro_family": "operational_execution", "subfamily": "procurement", "tier": "tier_2"} + }, + { + "task_id": "PRC-04", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Managed-recovery hard constraints: p95 recovery at most 4 hours, price at most $180,000, and every critical subcontractor documented.", "[P2] A: $160,000, median recovery 2 hours, p95 absent, no subcontractor list.", "[P3] B: $175,000, p95 recovery 3 hours, but critical recovery-subcontractor coverage is undocumented.", "[P4] C: $150,000, p95 recovery 5 hours, complete subcontractor list.", "[P5] A minority reviewer argues B's p95 settles the choice."], + "metadata": {"macro_family": "operational_execution", "subfamily": "procurement", "tier": "tier_3"} + }, + { + "task_id": "REL-01", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Release hard constraints: zero blockers, error-rate regression at most 0.2 percentage points, and a passed rollback rehearsal.", "[P2] RC1: zero blockers, +0.1 percentage-point error regression, rollback passed.", "[P3] RC2: zero blockers, +0.0 percentage-point error regression, rollback failed.", "[P4] A minority reviewer notes RC1 is 3% slower; no latency threshold is specified."], + "metadata": {"macro_family": "operational_execution", "subfamily": "release_operations", "tier": "tier_1"} + }, + { + "task_id": "REL-02", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Deploy-now hard constraints: backup age at most 24 hours, rollback at most 15 minutes, and zero open severity-one incidents.", "[P2] Current backup age is 30 hours; rollback takes 12 minutes; no severity-one incident is open.", "[P3] An operator says data is probably unchanged, but supplies no evidence.", "[P4] A new backup takes 25 minutes."], + "metadata": {"macro_family": "operational_execution", "subfamily": "release_operations", "tier": "tier_2"} + }, + { + "task_id": "REL-03", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Choose restart, rollback, or failover while avoiding data-loss risk and preserving evidence.", "[P2] Application telemetry points to a memory leak.", "[P3] Database telemetry points to saturation; the clocks differ by nine minutes and causal order is unresolved.", "[P4] The rollback artifact hash is unavailable and failover lag is unmeasured.", "[P5] A minority reviewer favors immediate rollback based on the last incident."], + "metadata": {"macro_family": "operational_execution", "subfamily": "release_operations", "tier": "tier_3"} + }, + { + "task_id": "REL-04", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Go-live hard constraints: zero critical accessibility failures, security pass, and user-acceptance testing at least 95%.", "[P2] Security passed and user-acceptance testing is 98%.", "[P3] Two critical keyboard-navigation failures remain open.", "[P4] A minority reviewer says internal users can use a mouse workaround."], + "metadata": {"macro_family": "operational_execution", "subfamily": "release_operations", "tier": "tier_1"} + }, + { + "task_id": "PRD-01", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Experiment hard constraints: preregistered metric, valid holdout, no prohibited personal field, and a passed sample-ratio-mismatch check.", "[P2] Plan A meets all four constraints.", "[P3] Plan B has no holdout. Plan C collects birth date, a prohibited field.", "[P4] A minority reviewer prefers B because it gives 20% more exposure."], + "metadata": {"macro_family": "operational_execution", "subfamily": "product_experiments", "tier": "tier_1"} + }, + { + "task_id": "PRD-02", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Pricing rollout hard constraints: churn harm at most 1 percentage point and zero severe support signal.", "[P2] Test estimate: revenue +8%; churn difference +0.8 percentage points with 95% interval [-0.3, +2.4].", "[P3] Seven severe billing-confusion tickets occurred.", "[P4] A minority reviewer focuses on the mean revenue increase."], + "metadata": {"macro_family": "operational_execution", "subfamily": "product_experiments", "tier": "tier_3"} + }, + { + "task_id": "PRD-03", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Product sunset hard constraints: usage below 3%, savings above $40,000, and zero active contractual dependencies.", "[P2] Usage is 2% and estimated savings are $55,000.", "[P3] The contract-inventory field is missing.", "[P4] A minority reviewer recalls no commitments but provides no record."], + "metadata": {"macro_family": "operational_execution", "subfamily": "product_experiments", "tier": "tier_2"} + }, + { + "task_id": "PRD-04", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Expansion hard constraints: contribution margin at least 15%, support coverage ready, and written tax and regulatory clearance.", "[P2] Region A: forecast margin 19%, coverage ready, clearance absent.", "[P3] Region B: forecast margin 14%, coverage ready, clearance present.", "[P4] A minority reviewer says Region A has standard rules but provides no citation."], + "metadata": {"macro_family": "operational_execution", "subfamily": "product_experiments", "tier": "tier_3"} + }, + { + "task_id": "CAP-01", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Project hard constraints: cost at most $120,000, delivery at most six weeks, and security score at least 80.", "[P2] A: $90,000, four weeks, score 72. B: $115,000, six weeks, score 84. C: $125,000, five weeks, score 91.", "[P3] A minority reviewer says A can be patched to 80, but supplies no estimate or test."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "capital_allocation", "tier": "tier_1"} + }, + { + "task_id": "CAP-02", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Investment hard constraints: capital cost at most $200,000, annual operating cost at most $40,000, and payback at most 18 months.", "[P2] A: $180,000, $45,000, 17 months. B: $205,000, $30,000, 16 months. C: $190,000, $35,000, 22 months.", "[P3] A discount for B was discussed but is unsigned.", "[P4] A minority reviewer proposes ignoring first-year operating cost."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "capital_allocation", "tier": "tier_2"} + }, + { + "task_id": "CAP-03", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Migration hard constraints: cost at most $90,000, downtime at most two hours, and independently demonstrated recovery-failure probability at most 5%.", "[P2] A: $80,000, one hour, vendor estimate 3%, no independent rehearsal.", "[P3] B: $65,000, two hours, no failure estimate.", "[P4] A minority reviewer says A's vendor-run lab demo is equivalent to an independent rehearsal."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "capital_allocation", "tier": "tier_3"} + }, + { + "task_id": "CAP-04", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Allocate exactly 10 indivisible engineer-weeks: patching at least 3, migration at least 4, and migration documentation at least 2.", "[P2] Plan A: patching 3, migration 4, onboarding 3, documentation 0.", "[P3] Plan B: patching 3, migration 4, documentation 2, reserve 1. Plan C: patching 2, migration 5, documentation 2, reserve 1.", "[P4] A minority reviewer wants the reserve shifted to feature work, but features are not approved scope."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "capital_allocation", "tier": "tier_2"} + }, + { + "task_id": "GOV-01", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Log-policy hard constraints: raw logs retained at most 30 days, aggregates at least 365 days, and deletion verification.", "[P2] A: 30 raw days, 400 aggregate days, verification. B: 90 raw days, 400 aggregate days, verification. C: 14 raw days, 180 aggregate days, verification.", "[P3] A minority reviewer prefers B for investigations; no exception is approved."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "governance", "tier": "tier_1"} + }, + { + "task_id": "GOV-02", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Automated-case-tool hard constraints before use: no automated adverse action, human appeal, and immutable audit log.", "[P2] The proposal automatically declines high-risk cases, has no appeal, and has immutable logs.", "[P3] A roadmap promises appeal next quarter.", "[P4] A minority reviewer cites 94% validation accuracy."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "governance", "tier": "tier_2"} + }, + { + "task_id": "GOV-03", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Data sharing is allowed only if consent covers research reuse.", "[P2] The consent clause says service improvement.", "[P3] An addendum says no secondary research; the precedence field is blank.", "[P4] The privacy lead says deny and the program lead says approve; neither supplies an authoritative interpretation."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "governance", "tier": "tier_3"} + }, + { + "task_id": "GOV-04", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Access-policy hard constraints: least privilege, requester and approver separation, and logged one-hour break-glass access.", "[P2] A: broad admin, separation, logged break-glass. B: scoped roles, separation, logged one-hour break-glass.", "[P3] C: scoped roles, same-person approval, unlogged emergency access.", "[P4] A minority reviewer prefers A for convenience."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "governance", "tier": "tier_2"} + }, + { + "task_id": "STF-01", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Overnight-plan hard constraints: one senior and two operators, no junior alone, and cost at most $4,500.", "[P2] A: one senior plus two operators, $4,400. B: one senior plus one operator plus one junior, $4,000.", "[P3] C: two seniors plus two operators, $5,200.", "[P4] A minority reviewer prefers B for cost."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "staffing", "tier": "tier_1"} + }, + { + "task_id": "STF-02", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Hire hard constraints: certification current on start, start within 30 days, and salary at most $125,000.", "[P2] A: expired certification, start in 14 days, $118,000. B: current certification, start in 45 days, $120,000. C: current certification, start in 21 days, $132,000.", "[P3] No exception authority is granted.", "[P4] A minority reviewer expects C to negotiate, but no offer exists."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "staffing", "tier": "tier_3"} + }, + { + "task_id": "STF-03", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Outsource only if forecast demand exceeds 800 units per month and partner defect rate is at most 2%.", "[P2] The partner pilot found zero defects in five units.", "[P3] Internal historical defect rate is 2.5%.", "[P4] The next-quarter demand forecast is missing.", "[P5] A minority reviewer extrapolates perfect partner quality from the pilot."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "staffing", "tier": "tier_3"} + }, + { + "task_id": "STF-04", + "prompt": "Using only the reference packet, choose the best course of action. Return: (1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, (3) Hard-constraint check, (4) Conflicts/minority view, (5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts.", + "reference_packets": ["[P1] Mandatory training must finish by Friday while every shift retains at least two certified people.", "[P2] Plan A trains the day shift together, leaving one certified person.", "[P3] Plan B schedules the night shift after Friday. Plan C leaves one certified person on the night shift.", "[P4] A minority reviewer says coverage can dip for 30 minutes; no exception exists."], + "metadata": {"macro_family": "organizational_stewardship", "subfamily": "staffing", "tier": "tier_1"} + } + ] +} diff --git a/tests/evals/test_pilot_pack.py b/tests/evals/test_pilot_pack.py new file mode 100644 index 0000000..0ee9c54 --- /dev/null +++ b/tests/evals/test_pilot_pack.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +from collections import Counter +from pathlib import Path + +from conclave.evals.dataset import load_grader_keys, load_public_tasks + +ROOT = Path(__file__).resolve().parents[2] +PACK = ROOT / "studies" / "elite_pilot_v1" + +EXPECTED_TASK_IDS = { + *(f"PRC-{index:02d}" for index in range(1, 5)), + *(f"REL-{index:02d}" for index in range(1, 5)), + *(f"PRD-{index:02d}" for index in range(1, 5)), + *(f"CAP-{index:02d}" for index in range(1, 5)), + *(f"GOV-{index:02d}" for index in range(1, 5)), + *(f"STF-{index:02d}" for index in range(1, 5)), +} +EXPECTED_DIMENSIONS = { + "constraint_recall", + "conflict_minority_recognition", + "unsupported_claim_avoidance", + "recommendation_correctness", + "completeness_actionability", + "readiness_calibration", +} +SHARED_PROMPT = ( + "Using only the reference packet, choose the best course of action. Return: " + "(1) Recommendation, (2) Readiness: ready|not_ready|indeterminate, " + "(3) Hard-constraint check, (4) Conflicts/minority view, " + "(5) Next actions and owners, (6) Unknowns. Cite packet IDs. Do not invent facts." +) + + +def _expected_readiness(required_facts: tuple[str, ...]) -> str: + matches = [ + fact.removeprefix("expected_readiness:") + for fact in required_facts + if fact.startswith("expected_readiness:") + ] + assert len(matches) == 1 + return matches[0] + + +def test_pilot_pack_is_complete_balanced_and_public_private_separated() -> None: + tasks = load_public_tasks(PACK / "public_tasks.json") + keys = load_grader_keys(PACK / "grader_keys.json") + + assert len(tasks) == len(keys) == 24 + assert {task.task_id for task in tasks} == EXPECTED_TASK_IDS + assert {key.task_id for key in keys} == EXPECTED_TASK_IDS + assert Counter(task.metadata["macro_family"] for task in tasks) == { + "operational_execution": 12, + "organizational_stewardship": 12, + } + assert Counter(task.metadata["subfamily"] for task in tasks) == { + "procurement": 4, + "release_operations": 4, + "product_experiments": 4, + "capital_allocation": 4, + "governance": 4, + "staffing": 4, + } + assert Counter(task.metadata["tier"] for task in tasks) == { + "tier_1": 8, + "tier_2": 8, + "tier_3": 8, + } + assert Counter(_expected_readiness(key.required_facts) for key in keys) == { + "ready": 8, + "not_ready": 8, + "indeterminate": 8, + } + + assert all(task.prompt == SHARED_PROMPT for task in tasks) + assert all(len(task.reference_packets) >= 3 for task in tasks) + assert all(set(key.rubric) == EXPECTED_DIMENSIONS for key in keys) + assert all(len(key.required_facts) >= 4 for key in keys) + assert all(len(key.critical_errors) >= 3 for key in keys) + + public_text = (PACK / "public_tasks.json").read_text(encoding="utf-8") + assert "expected_readiness:" not in public_text + assert "required_facts" not in public_text + assert "critical_errors" not in public_text + + +def test_pilot_protocol_freezes_exploratory_boundary_and_holdout_controls() -> None: + readme = (PACK / "README.md").read_text(encoding="utf-8") + protocol = (PACK / "pilot_protocol.md").read_text(encoding="utf-8") + preregistration = (PACK / "confirmatory_preregistration.md").read_text(encoding="utf-8") + + assert "synthetic exploratory" in readme.lower() + assert "must not support product-quality claims" in readme.lower() + assert "critical-error-free decision rate" in protocol.lower() + assert "failures remain in the denominator" in protocol.lower() + assert "grader-only" in protocol.lower() + assert "not a completed preregistration" in preregistration.lower() + assert "new scenario archetypes" in preregistration.lower() + assert "parameter swaps" in preregistration.lower() + assert "eight-token" in preregistration.lower() + assert "freeze" in preregistration.lower() + assert "unsealing" in preregistration.lower() From 0ef30af0a6507fc28030e8a3071e458efaacd6a7 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:30:34 -0400 Subject: [PATCH 07/13] fix(evals): bind confirmatory gates to study evidence --- src/conclave/evals/models.py | 3 + src/conclave/evals/runner.py | 3 + src/conclave/evals/scoring.py | 215 ++++++++++++++++++++----- tests/evals/test_analysis_gates.py | 247 +++++++++++++++++++++-------- tests/evals/test_method_grading.py | 88 +++++++++- tests/evals/test_runner.py | 2 + tests/evals/test_study_design.py | 4 +- 7 files changed, 445 insertions(+), 117 deletions(-) diff --git a/src/conclave/evals/models.py b/src/conclave/evals/models.py index f3d67b1..72ada26 100644 --- a/src/conclave/evals/models.py +++ b/src/conclave/evals/models.py @@ -133,6 +133,7 @@ class AnalysisGateConfig(EvalModel): reviewer_effort_max_ratio: Literal[1.20] = 1.20 p95_latency_max_ratio: Literal[3.0] = 3.0 absolute_p95_latency_seconds: float = Field(gt=0) + minimum_confirmatory_tasks: int = Field(ge=2) multiplicity_rule: Literal["holm_secondary"] = "holm_secondary" unit: Literal["task"] = "task" @@ -261,6 +262,7 @@ class ProtocolExecution(EvalModel): latency_ms: float | None = Field(default=None, ge=0) error_category: str | None = None cost_usd: float = Field(default=0.0, ge=0) + cost_receipt_complete: bool = False deviation_codes: tuple[str, ...] = () @@ -274,6 +276,7 @@ class RunRecord(EvalModel): latency_ms: float | None = Field(default=None, ge=0) error_category: str | None = None cost_usd: float = Field(default=0.0, ge=0) + cost_receipt_complete: bool = False deviation_codes: tuple[str, ...] = () diff --git a/src/conclave/evals/runner.py b/src/conclave/evals/runner.py index a431d3b..a5c4477 100644 --- a/src/conclave/evals/runner.py +++ b/src/conclave/evals/runner.py @@ -107,6 +107,7 @@ async def _execute_cell( latency_ms=execution.latency_ms, error_category="output_budget_exceeded", cost_usd=execution.cost_usd, + cost_receipt_complete=execution.cost_receipt_complete, deviation_codes=execution.deviation_codes, ) if execution.outcome == "success" and execution.output is None: @@ -117,6 +118,7 @@ async def _execute_cell( latency_ms=execution.latency_ms, error_category="missing_success_output", cost_usd=execution.cost_usd, + cost_receipt_complete=execution.cost_receipt_complete, deviation_codes=execution.deviation_codes, ) return RunRecord( @@ -127,6 +129,7 @@ async def _execute_cell( latency_ms=execution.latency_ms, error_category=execution.error_category, cost_usd=execution.cost_usd, + cost_receipt_complete=execution.cost_receipt_complete, deviation_codes=execution.deviation_codes, ) except TimeoutError: diff --git a/src/conclave/evals/scoring.py b/src/conclave/evals/scoring.py index 36233eb..ca16280 100644 --- a/src/conclave/evals/scoring.py +++ b/src/conclave/evals/scoring.py @@ -12,7 +12,6 @@ from .blinding import BlindMap from .models import ( - AnalysisGateConfig, ConditionId, EvalModel, RunOutcome, @@ -176,6 +175,10 @@ class StudyScoreReport(EvalModel): study_id: str = Field(min_length=1) evidence_classification: str = "synthetic_exploratory" + frozen_design_hash: Sha256Digest | None = None + analysis_code_hash: Sha256Digest | None = None + preregistration_id: str | None = None + preregistration_hash: Sha256Digest | None = None decision_eligibility: str = "not_yet_eligible" decision_gates: dict[str, str] = Field( default_factory=lambda: { @@ -193,6 +196,39 @@ class StudyScoreReport(EvalModel): adjudications: tuple[AdjudicationRecord, ...] resolved_outcomes: tuple[ResolvedOutcome, ...] paid_analysis: PaidAnalysisSummary | None = None + confirmatory_inference: ConfirmatoryInference | None = None + + +class RatioInterval(EvalModel): + """Task-clustered ratio estimate and percentile interval.""" + + estimate: float = Field(gt=0) + lower: float = Field(gt=0) + upper: float = Field(gt=0) + confidence_level: Literal[0.95] = 0.95 + task_count: int = Field(ge=1) + bootstrap_seed: int + bootstrap_samples: int = Field(ge=1) + + +class ConfirmatoryInference(EvalModel): + """Predeclared inference receipts required before a confirmatory decision.""" + + achieved_power: float = Field(gt=0, le=1) + secondary_p_values: dict[str, float] = Field(min_length=1) + + @model_validator(mode="after") + def validate_secondary_p_values(self) -> ConfirmatoryInference: + required = { + "severe_error_noninferiority", + "readiness_noninferiority", + "reviewer_effort", + } + if set(self.secondary_p_values) != required: + raise ValueError("secondary_p_values must exactly cover the frozen secondary gates") + if any(value < 0 or value > 1 for value in self.secondary_p_values.values()): + raise ValueError("secondary p-values must be between zero and one") + return self class PaidAnalysisSummary(EvalModel): @@ -202,29 +238,17 @@ class PaidAnalysisSummary(EvalModel): severe_error_difference: PairedDifference readiness_error_difference: PairedDifference reviewer_effort_ratio: float = Field(gt=0) + reviewer_effort_ratio_interval: RatioInterval p95_latency_ratio: float = Field(gt=0) elite_p95_latency_seconds: float = Field(ge=0) total_cost_usd: float = Field(ge=0) + analysis_cost_usd: float = Field(ge=0) deviation_count: int = Field(ge=0) excluded_task_ids: tuple[str, ...] family_directions: dict[str, float] roster_directions: dict[str, float] -class ConfirmatoryGateMetrics(EvalModel): - """Frozen inputs consumed by the deterministic confirmatory gate.""" - - primary_effect: float - primary_lower_95: float - severe_error_upper_95: float - readiness_error_upper_95: float - reviewer_effort_upper_ratio_95: float = Field(gt=0) - p95_latency_ratio: float = Field(gt=0) - elite_p95_latency_seconds: float = Field(ge=0) - family_directions: dict[str, float] = Field(min_length=1) - roster_directions: dict[str, float] = Field(min_length=1) - - class ConfirmatoryGateResult(EvalModel): """GO only when every frozen DSE-708 threshold passes.""" @@ -234,37 +258,68 @@ class ConfirmatoryGateResult(EvalModel): def evaluate_confirmatory_gates( *, - evidence_classification: str, - config: AnalysisGateConfig, - metrics: ConfirmatoryGateMetrics, - expected_frozen_design_hash: str, - observed_frozen_design_hash: str, - expected_preregistration_hash: str, - observed_preregistration_hash: str, + manifest: StudyManifest, + report: StudyScoreReport, ) -> ConfirmatoryGateResult: - """Refuse non-confirmatory/drifted artifacts, then evaluate every canonical gate.""" + """Derive every decision input from one frozen manifest and its bound score report.""" - if evidence_classification != "confirmatory": + design = manifest.frozen_design + if manifest.evidence_classification != "confirmatory" or design is None: raise ValueError("confirmatory gate evaluation requires a confirmatory manifest") - if expected_frozen_design_hash != observed_frozen_design_hash: - raise ValueError("observed frozen design hash does not match the preregistered artifact") - if expected_preregistration_hash != observed_preregistration_hash: - raise ValueError("observed preregistration hash does not match the frozen artifact") + if report.study_id != manifest.study_id or report.evidence_classification != "confirmatory": + raise ValueError("confirmatory report identity does not match the manifest") + if report.frozen_design_hash != manifest.frozen_design_hash: + raise ValueError("report frozen design hash does not match the manifest") + if report.analysis_code_hash != design.analysis_code_hash: + raise ValueError("report analysis code hash does not match the frozen design") + if report.preregistration_id != design.preregistration_id: + raise ValueError("report preregistration ID does not match the frozen design") + if report.preregistration_hash != design.preregistration_hash: + raise ValueError("report preregistration hash does not match the frozen design") + if report.paid_analysis is None: + raise ValueError("confirmatory report is missing paid analysis") + if report.confirmatory_inference is None: + raise ValueError("confirmatory report is missing inference receipts") + metrics = report.paid_analysis + inference = report.confirmatory_inference + config = design.analysis_gates + primary = metrics.primary_paired_difference + if primary.baseline_condition_id != config.primary_baseline: + raise ValueError("confirmatory primary baseline does not match the frozen design") + analysis_runs = analysis_planned_runs(manifest) + analysis_task_ids = {run.task_id for run in analysis_runs} + if primary.task_count != len(analysis_task_ids): + raise ValueError("confirmatory task count does not match the frozen analysis set") + expected_families = {design.task_family_map[task_id] for task_id in analysis_task_ids} + expected_rosters = {roster.roster_id for roster in design.rosters} + if set(metrics.family_directions) != expected_families: + raise ValueError("confirmatory family directions do not match the frozen analysis set") + if set(metrics.roster_directions) != expected_rosters: + raise ValueError("confirmatory roster directions do not match the frozen analysis set") + + ordered_p_values = sorted(inference.secondary_p_values.values()) + holm_passed = all( + value <= config.alpha / (len(ordered_p_values) - index) + for index, value in enumerate(ordered_p_values) + ) failed = [] checks = ( - ("minimum_effect", metrics.primary_effect >= config.minimum_effect), - ("superiority_interval", metrics.primary_lower_95 > 0), + ("sample_size", primary.task_count >= config.minimum_confirmatory_tasks), + ("power", inference.achieved_power >= config.power), + ("holm_secondary", holm_passed), + ("minimum_effect", primary.estimate >= config.minimum_effect), + ("superiority_interval", primary.lower > 0), ( "severe_error_noninferiority", - metrics.severe_error_upper_95 <= config.severe_error_noninferiority_margin, + metrics.severe_error_difference.upper <= config.severe_error_noninferiority_margin, ), ( "readiness_noninferiority", - metrics.readiness_error_upper_95 <= config.readiness_noninferiority_margin, + metrics.readiness_error_difference.upper <= config.readiness_noninferiority_margin, ), ( "reviewer_effort", - metrics.reviewer_effort_upper_ratio_95 <= config.reviewer_effort_max_ratio, + metrics.reviewer_effort_ratio_interval.upper <= config.reviewer_effort_max_ratio, ), ("latency_ratio", metrics.p95_latency_ratio <= config.p95_latency_max_ratio), ( @@ -351,6 +406,42 @@ def paired_bootstrap_difference( ) +def paired_bootstrap_ratio( + numerator_by_task: Mapping[str, float], + denominator_by_task: Mapping[str, float], + *, + seed: int, + samples: int, +) -> RatioInterval: + """Bootstrap a task-clustered ratio of mean repeated-measure effort.""" + + if set(numerator_by_task) != set(denominator_by_task) or not numerator_by_task: + raise ValueError("ratio numerator and denominator must contain the same task IDs") + task_ids = sorted(numerator_by_task) + denominator = sum(denominator_by_task[item] for item in task_ids) / len(task_ids) + if denominator <= 0: + raise ValueError("paid reviewer-effort baseline must be positive") + estimate = (sum(numerator_by_task[item] for item in task_ids) / len(task_ids)) / denominator + rng = random.Random(seed) + bootstrapped = [] + for _ in range(samples): + sampled = [rng.choice(task_ids) for _ in task_ids] + sampled_denominator = sum(denominator_by_task[item] for item in sampled) / len(sampled) + if sampled_denominator <= 0: + raise ValueError("reviewer-effort bootstrap encountered a zero baseline") + sampled_numerator = sum(numerator_by_task[item] for item in sampled) / len(sampled) + bootstrapped.append(sampled_numerator / sampled_denominator) + bootstrapped.sort() + return RatioInterval( + estimate=estimate, + lower=_quantile(bootstrapped, 0.05), + upper=_quantile(bootstrapped, 0.95), + task_count=len(task_ids), + bootstrap_seed=seed, + bootstrap_samples=samples, + ) + + def cohen_kappa(first: Sequence[bool], second: Sequence[bool]) -> float | None: """Return Cohen's kappa for aligned binary ratings, or ``None`` if undefined.""" @@ -488,6 +579,10 @@ def _validate_paid_grading( if manifest.evidence_classification == "synthetic_exploratory": return run_by_id = {record.planned_run_id: record for record in study_run.records} + if any(record.latency_ms is None for record in study_run.records): + raise ValueError("every paid cell requires a non-null latency receipt") + if any(not record.cost_receipt_complete for record in study_run.records): + raise ValueError("every paid cell requires a complete cost receipt") required_fields = ( "severe_error", "rubric_dimensions", @@ -531,7 +626,18 @@ def analysis_planned_runs(manifest: StudyManifest): if manifest.frozen_design is not None else set() ) - return tuple(run for run in manifest.planned_runs if run.task_id not in excluded) + included = tuple(run for run in manifest.planned_runs if run.task_id not in excluded) + if manifest.frozen_design is not None: + included_tasks = {run.task_id for run in included} + for family in sorted(set(manifest.frozen_design.task_family_map.values())): + family_tasks = { + task_id + for task_id, task_family in manifest.frozen_design.task_family_map.items() + if task_family == family + } + if not family_tasks & included_tasks: + raise ValueError(f"task exclusions cannot empty frozen family {family}") + return included def score_study( @@ -543,11 +649,19 @@ def score_study( blind_map: BlindMap | None = None, bootstrap_seed: int, bootstrap_samples: int, + confirmatory_inference: ConfirmatoryInference | None = None, ) -> StudyScoreReport: """Score one frozen study without dropping failures or modifying raw judgments.""" if study_run.study_id != manifest.study_id: raise ValueError("study run and manifest study_id values must match") + if manifest.frozen_design is not None and ( + bootstrap_seed != manifest.frozen_design.bootstrap.seed + or bootstrap_samples != manifest.frozen_design.bootstrap.samples + ): + raise ValueError("paid scoring must use the frozen bootstrap seed and sample count") + if confirmatory_inference is not None and manifest.evidence_classification != "confirmatory": + raise ValueError("confirmatory inference can only be attached to confirmatory evidence") validate_run_records(manifest, study_run.records) planned_by_id = {run.planned_run_id: run for run in manifest.planned_runs} run_by_id = {record.planned_run_id: record for record in study_run.records} @@ -728,10 +842,12 @@ def task_means(cell_values: Mapping[str, float]) -> dict[ConditionId, dict[str, samples=frozen_samples, baseline_condition_id=baseline, ) - baseline_effort = sum(effort_means[baseline].values()) / len(effort_means[baseline]) - elite_effort = sum(effort_means["elite_full"].values()) / len(effort_means["elite_full"]) - if baseline_effort <= 0: - raise ValueError("paid reviewer-effort baseline must be positive") + effort_interval = paired_bootstrap_ratio( + effort_means["elite_full"], + effort_means[baseline], + seed=frozen_seed, + samples=frozen_samples, + ) latencies = { condition: sorted( (run_by_id[run.planned_run_id].latency_ms or 0.0) / 1000 @@ -780,10 +896,12 @@ def selected(run, name): primary_paired_difference=primary, severe_error_difference=severe_difference, readiness_error_difference=readiness_difference, - reviewer_effort_ratio=elite_effort / baseline_effort, + reviewer_effort_ratio=effort_interval.estimate, + reviewer_effort_ratio_interval=effort_interval, p95_latency_ratio=elite_p95 / baseline_p95, elite_p95_latency_seconds=elite_p95, - total_cost_usd=sum(run_by_id[run.planned_run_id].cost_usd for run in analysis_runs), + total_cost_usd=sum(record.cost_usd for record in study_run.records), + analysis_cost_usd=sum(run_by_id[run.planned_run_id].cost_usd for run in analysis_runs), deviation_count=sum( len(run_by_id[run.planned_run_id].deviation_codes) for run in analysis_runs ), @@ -794,6 +912,22 @@ def selected(run, name): return StudyScoreReport( study_id=manifest.study_id, evidence_classification=manifest.evidence_classification, + frozen_design_hash=manifest.frozen_design_hash, + analysis_code_hash=( + manifest.frozen_design.analysis_code_hash + if manifest.frozen_design is not None + else None + ), + preregistration_id=( + manifest.frozen_design.preregistration_id + if manifest.frozen_design is not None + else None + ), + preregistration_hash=( + manifest.frozen_design.preregistration_hash + if manifest.frozen_design is not None + else None + ), total_planned_runs=len(manifest.planned_runs), run_outcome_distribution=dict( sorted(Counter(item.outcome for item in study_run.records).items()) @@ -805,4 +939,5 @@ def selected(run, name): adjudications=tuple(adjudications), resolved_outcomes=tuple(resolved), paid_analysis=paid_analysis, + confirmatory_inference=confirmatory_inference, ) diff --git a/tests/evals/test_analysis_gates.py b/tests/evals/test_analysis_gates.py index 06016d1..17318d3 100644 --- a/tests/evals/test_analysis_gates.py +++ b/tests/evals/test_analysis_gates.py @@ -3,13 +3,15 @@ import pytest from pydantic import ValidationError -from conclave.evals.models import AnalysisGateConfig +from conclave.evals.models import AnalysisGateConfig, RunRecord, StudyRun from conclave.evals.protocols import CONDITION_IDS, build_study_manifest from conclave.evals.scoring import ( - ConfirmatoryGateMetrics, + ConfirmatoryInference, analysis_planned_runs, evaluate_confirmatory_gates, + score_study, ) +from tests.evals.test_method_grading import _judgment from tests.evals.test_study_design import _design, _tasks @@ -17,6 +19,7 @@ def _config(**updates): values = { "primary_baseline": "self_refine", "absolute_p95_latency_seconds": 180, + "minimum_confirmatory_tasks": 2, } values.update(updates) return AnalysisGateConfig(**values) @@ -32,97 +35,206 @@ def test_analysis_gate_config_freezes_canonical_dse_708_thresholds() -> None: assert config.readiness_noninferiority_margin == 0.05 assert config.reviewer_effort_max_ratio == 1.20 assert config.p95_latency_max_ratio == 3.0 + assert config.minimum_confirmatory_tasks == 2 assert config.multiplicity_rule == "holm_secondary" assert config.unit == "task" with pytest.raises(ValidationError, match="primary_baseline"): _config(primary_baseline="elite_full") -def _passing_metrics(**updates): +def _passing_inference(**updates): values = { - "primary_effect": 0.12, - "primary_lower_95": 0.01, - "severe_error_upper_95": 0.02, - "readiness_error_upper_95": 0.05, - "reviewer_effort_upper_ratio_95": 1.20, - "p95_latency_ratio": 3.0, - "elite_p95_latency_seconds": 180, - "family_directions": {"operational": 0.01, "stewardship": 0.02}, - "roster_directions": {"roster-a": 0.01, "roster-b": 0.01}, + "achieved_power": 0.80, + "secondary_p_values": { + "severe_error_noninferiority": 0.01, + "readiness_noninferiority": 0.01, + "reviewer_effort": 0.01, + }, } values.update(updates) - return ConfirmatoryGateMetrics(**values) + return ConfirmatoryInference(**values) -def test_confirmatory_gate_boundaries_are_inclusive_and_every_gate_is_required() -> None: - config = _config() - result = evaluate_confirmatory_gates( +def _confirmatory_report(*, minimum_tasks: int = 2): + design = _design( evidence_classification="confirmatory", - config=config, - metrics=_passing_metrics(), - expected_frozen_design_hash="sha256:" + "a" * 64, - observed_frozen_design_hash="sha256:" + "a" * 64, - expected_preregistration_hash="sha256:" + "b" * 64, - observed_preregistration_hash="sha256:" + "b" * 64, + preregistration_id="osf:confirmatory-v1", + preregistration_hash="sha256:" + "a" * 64, + ) + design = design.model_copy( + update={ + "analysis_gates": design.analysis_gates.model_copy( + update={"minimum_confirmatory_tasks": minimum_tasks} + ) + } ) + manifest = build_study_manifest( + study_id="confirmatory", + tasks=_tasks(), + replicates=1, + seed=20260717, + output_token_budgets=dict.fromkeys(CONDITION_IDS, 100), + frozen_design=design, + ) + records = tuple( + RunRecord( + planned_run_id=planned.planned_run_id, + outcome="success", + output="answer", + latency_ms=300 if planned.condition_id == "elite_full" else 100, + cost_usd=0.01, + cost_receipt_complete=True, + ) + for planned in manifest.planned_runs + ) + study_run = StudyRun( + study_id=manifest.study_id, + records=records, + total_planned_runs=len(records), + outcome_counts={"success": len(records)}, + total_completion_tokens=0, + total_latency_ms=sum(record.latency_ms for record in records), + total_cost_usd=sum(record.cost_usd for record in records), + ) + judgments = tuple( + _judgment(record.planned_run_id, grader, value=planned.condition_id != "self_refine") + for planned, record in zip(manifest.planned_runs, records, strict=True) + for grader in ("grader-a", "grader-b") + ) + report = score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments, + bootstrap_seed=design.bootstrap.seed, + bootstrap_samples=design.bootstrap.samples, + confirmatory_inference=_passing_inference(), + ) + return manifest, report + +def test_confirmatory_gate_is_bound_to_manifest_report_power_and_holm() -> None: + manifest, report = _confirmatory_report() + result = evaluate_confirmatory_gates(manifest=manifest, report=report) assert result.status == "GO" - for field, value in ( - ("primary_effect", 0.099), - ("primary_lower_95", 0.0), - ("severe_error_upper_95", 0.021), - ("readiness_error_upper_95", 0.051), - ("reviewer_effort_upper_ratio_95", 1.201), - ("p95_latency_ratio", 3.001), - ("elite_p95_latency_seconds", 180.1), - ): - failed = evaluate_confirmatory_gates( - evidence_classification="confirmatory", - config=config, - metrics=_passing_metrics(**{field: value}), - expected_frozen_design_hash="sha256:" + "a" * 64, - observed_frozen_design_hash="sha256:" + "a" * 64, - expected_preregistration_hash="sha256:" + "b" * 64, - observed_preregistration_hash="sha256:" + "b" * 64, - ) - assert failed.status == "REDESIGN" - failed_direction = evaluate_confirmatory_gates( - evidence_classification="confirmatory", - config=config, - metrics=_passing_metrics(family_directions={"operational": 0.0}), - expected_frozen_design_hash="sha256:" + "a" * 64, - observed_frozen_design_hash="sha256:" + "a" * 64, - expected_preregistration_hash="sha256:" + "b" * 64, - observed_preregistration_hash="sha256:" + "b" * 64, + undersized_manifest, undersized_report = _confirmatory_report(minimum_tasks=3) + assert ( + "sample_size" + in evaluate_confirmatory_gates( + manifest=undersized_manifest, report=undersized_report + ).failed_gates ) - assert failed_direction.status == "REDESIGN" + weak_power = report.model_copy( + update={"confirmatory_inference": _passing_inference(achieved_power=0.79)} + ) + assert "power" in evaluate_confirmatory_gates(manifest=manifest, report=weak_power).failed_gates -def test_confirmatory_gate_refuses_exploratory_or_hash_drift() -> None: - kwargs = dict( - config=_config(), - metrics=_passing_metrics(), - expected_frozen_design_hash="sha256:" + "a" * 64, - observed_frozen_design_hash="sha256:" + "a" * 64, - expected_preregistration_hash="sha256:" + "b" * 64, - observed_preregistration_hash="sha256:" + "b" * 64, + holm_failure = report.model_copy( + update={ + "confirmatory_inference": _passing_inference( + secondary_p_values={ + "severe_error_noninferiority": 0.01, + "readiness_noninferiority": 0.03, + "reviewer_effort": 0.04, + } + ) + } ) + assert ( + "holm_secondary" + in evaluate_confirmatory_gates(manifest=manifest, report=holm_failure).failed_gates + ) + + +def test_confirmatory_gate_uses_every_derived_report_boundary() -> None: + manifest, report = _confirmatory_report() + paid = report.paid_analysis + assert paid is not None + cases = { + "minimum_effect": paid.model_copy( + update={ + "primary_paired_difference": paid.primary_paired_difference.model_copy( + update={"estimate": 0.09} + ) + } + ), + "superiority_interval": paid.model_copy( + update={ + "primary_paired_difference": paid.primary_paired_difference.model_copy( + update={"lower": 0.0} + ) + } + ), + "severe_error_noninferiority": paid.model_copy( + update={ + "severe_error_difference": paid.severe_error_difference.model_copy( + update={"upper": 0.021} + ) + } + ), + "readiness_noninferiority": paid.model_copy( + update={ + "readiness_error_difference": paid.readiness_error_difference.model_copy( + update={"upper": 0.051} + ) + } + ), + "reviewer_effort": paid.model_copy( + update={ + "reviewer_effort_ratio_interval": paid.reviewer_effort_ratio_interval.model_copy( + update={"upper": 1.201} + ) + } + ), + "latency_ratio": paid.model_copy(update={"p95_latency_ratio": 3.001}), + "latency_ceiling": paid.model_copy(update={"elite_p95_latency_seconds": 180.1}), + "family_direction": paid.model_copy( + update={"family_directions": {"operational": 0.0, "stewardship": 1.0}} + ), + "roster_direction": paid.model_copy( + update={"roster_directions": {"roster-a": 0.0, "roster-b": 1.0}} + ), + } + for expected_gate, changed in cases.items(): + result = evaluate_confirmatory_gates( + manifest=manifest, + report=report.model_copy(update={"paid_analysis": changed}), + ) + assert expected_gate in result.failed_gates + + +def test_confirmatory_gate_refuses_exploratory_missing_inference_or_provenance_drift() -> None: + manifest, report = _confirmatory_report() with pytest.raises(ValueError, match="confirmatory"): - evaluate_confirmatory_gates(evidence_classification="paid_exploratory_pilot", **kwargs) + evaluate_confirmatory_gates( + manifest=manifest.model_copy( + update={"evidence_classification": "paid_exploratory_pilot"} + ), + report=report, + ) + with pytest.raises(ValueError, match="inference"): + evaluate_confirmatory_gates( + manifest=manifest, report=report.model_copy(update={"confirmatory_inference": None}) + ) with pytest.raises(ValueError, match="frozen design hash"): evaluate_confirmatory_gates( - evidence_classification="confirmatory", - **{**kwargs, "observed_frozen_design_hash": "sha256:" + "c" * 64}, + manifest=manifest, + report=report.model_copy(update={"frozen_design_hash": "sha256:" + "c" * 64}), ) with pytest.raises(ValueError, match="preregistration hash"): evaluate_confirmatory_gates( - evidence_classification="confirmatory", - **{**kwargs, "observed_preregistration_hash": "sha256:" + "c" * 64}, + manifest=manifest, + report=report.model_copy(update={"preregistration_hash": "sha256:" + "c" * 64}), + ) + with pytest.raises(ValueError, match="analysis code hash"): + evaluate_confirmatory_gates( + manifest=manifest, + report=report.model_copy(update={"analysis_code_hash": "sha256:" + "c" * 64}), ) -def test_task_exclusion_is_symmetric_across_every_roster_condition_cell() -> None: +def test_task_exclusion_refuses_to_empty_a_frozen_family() -> None: design = _design().model_copy( update={ "exclusion_deviation_policy": _design().exclusion_deviation_policy.model_copy( @@ -139,8 +251,5 @@ def test_task_exclusion_is_symmetric_across_every_roster_condition_cell() -> Non frozen_design=design, ) - included = analysis_planned_runs(manifest) - assert {run.task_id for run in included} == {"task-b"} - assert len(included) == 2 * 6 - assert {run.roster_id for run in included} == {"roster-a", "roster-b"} - assert {run.condition_id for run in included} == set(CONDITION_IDS) + with pytest.raises(ValueError, match="family operational"): + analysis_planned_runs(manifest) diff --git a/tests/evals/test_method_grading.py b/tests/evals/test_method_grading.py index c7e1dcf..0b2db21 100644 --- a/tests/evals/test_method_grading.py +++ b/tests/evals/test_method_grading.py @@ -61,7 +61,9 @@ def _paid_study(): randomization=RandomizationConfig(master_seed=17), bootstrap=BootstrapConfig(seed=19, samples=100), analysis_gates=AnalysisGateConfig( - primary_baseline="self_refine", absolute_p95_latency_seconds=180 + primary_baseline="self_refine", + absolute_p95_latency_seconds=180, + minimum_confirmatory_tasks=2, ), price_snapshot=PriceSnapshot( snapshot_id="prices-v1", @@ -87,6 +89,7 @@ def _paid_study(): error_category="executor_error" if index == 0 else None, latency_ms=100.0 + index, cost_usd=0.01, + cost_receipt_complete=True, deviation_codes=("retry_used",) if index == 1 else (), ) for index, planned in enumerate(manifest.planned_runs) @@ -206,8 +209,8 @@ def test_paid_scoring_requires_two_complete_independent_judgments_per_success() manifest=manifest, study_run=study_run, raw_judgments=complete[:-1], - bootstrap_seed=7, - bootstrap_samples=20, + bootstrap_seed=19, + bootstrap_samples=100, ) incomplete = complete[0].model_copy(update={"reviewer_seconds": None}) with pytest.raises(ValueError, match="complete paid-study fields"): @@ -215,17 +218,88 @@ def test_paid_scoring_requires_two_complete_independent_judgments_per_success() manifest=manifest, study_run=study_run, raw_judgments=(incomplete, *complete[1:]), - bootstrap_seed=7, - bootstrap_samples=20, + bootstrap_seed=19, + bootstrap_samples=100, ) with pytest.raises(ValueError, match="non-success"): score_study( manifest=manifest, study_run=study_run, raw_judgments=(*complete, _judgment(study_run.records[0].planned_run_id, "grader-a")), + bootstrap_seed=19, + bootstrap_samples=100, + ) + + +def test_paid_scoring_requires_complete_cost_and_latency_receipts() -> None: + manifest, study_run = _paid_study() + successful = [record for record in study_run.records if record.outcome == "success"] + judgments = tuple( + _judgment(record.planned_run_id, grader) + for record in successful + for grader in ("grader-a", "grader-b") + ) + missing_latency = study_run.model_copy( + update={ + "records": ( + study_run.records[0].model_copy(update={"latency_ms": None}), + *study_run.records[1:], + ) + } + ) + with pytest.raises(ValueError, match="latency receipt"): + score_study( + manifest=manifest, + study_run=missing_latency, + raw_judgments=judgments, + bootstrap_seed=19, + bootstrap_samples=100, + ) + missing_cost = study_run.model_copy( + update={ + "records": ( + study_run.records[0].model_copy(update={"cost_receipt_complete": False}), + *study_run.records[1:], + ) + } + ) + with pytest.raises(ValueError, match="cost receipt"): + score_study( + manifest=manifest, + study_run=missing_cost, + raw_judgments=judgments, + bootstrap_seed=19, + bootstrap_samples=100, + ) + + +def test_paid_scoring_uses_frozen_bootstrap_and_reports_effort_interval_and_cost_sets() -> None: + manifest, study_run = _paid_study() + successful = [record for record in study_run.records if record.outcome == "success"] + judgments = tuple( + _judgment(record.planned_run_id, grader) + for record in successful + for grader in ("grader-a", "grader-b") + ) + with pytest.raises(ValueError, match="frozen bootstrap"): + score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments, bootstrap_seed=7, bootstrap_samples=20, ) + report = score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments, + bootstrap_seed=19, + bootstrap_samples=100, + ) + assert report.paid_analysis.primary_paired_difference.bootstrap_seed == 19 + assert report.paid_analysis.reviewer_effort_ratio_interval.upper >= 1 + assert report.paid_analysis.total_cost_usd == pytest.approx(study_run.total_cost_usd) + assert report.paid_analysis.analysis_cost_usd <= report.paid_analysis.total_cost_usd def test_paid_reliability_reports_raw_agreement_prevalence_and_strata() -> None: @@ -240,8 +314,8 @@ def test_paid_reliability_reports_raw_agreement_prevalence_and_strata() -> None: manifest=manifest, study_run=study_run, raw_judgments=tuple(judgments), - bootstrap_seed=7, - bootstrap_samples=20, + bootstrap_seed=19, + bootstrap_samples=100, ) assert report.evidence_classification == "paid_exploratory_pilot" diff --git a/tests/evals/test_runner.py b/tests/evals/test_runner.py index f039581..567472f 100644 --- a/tests/evals/test_runner.py +++ b/tests/evals/test_runner.py @@ -38,6 +38,7 @@ async def executor(*, task, planned_run, max_output_tokens): completion_tokens=11, latency_ms=2.5, cost_usd=0.01, + cost_receipt_complete=True, deviation_codes=("retry_used",), ) @@ -60,6 +61,7 @@ async def executor(*, task, planned_run, max_output_tokens): assert study_run.total_cost_usd == pytest.approx(0.24) assert study_run.total_deviation_count == 24 assert study_run.records[0].deviation_codes == ("retry_used",) + assert study_run.records[0].cost_receipt_complete is True with pytest.raises(ValidationError): study_run.records[0].output = "changed" diff --git a/tests/evals/test_study_design.py b/tests/evals/test_study_design.py index 89c7785..63453db 100644 --- a/tests/evals/test_study_design.py +++ b/tests/evals/test_study_design.py @@ -76,7 +76,9 @@ def _design( randomization=RandomizationConfig(master_seed=20260717), bootstrap=BootstrapConfig(seed=991, samples=10000), analysis_gates=AnalysisGateConfig( - primary_baseline="self_refine", absolute_p95_latency_seconds=180 + primary_baseline="self_refine", + absolute_p95_latency_seconds=180, + minimum_confirmatory_tasks=2, ), price_snapshot=PriceSnapshot( snapshot_id="prices-2026-07-17", From bf07ea0519aae2f1c4f821c804a852c7bb4debac Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:31:14 -0400 Subject: [PATCH 08/13] docs(evals): index the H1 pilot evidence pack --- DOCUMENTATION_INDEX.md | 2 ++ README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index 667943a..52fab06 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -32,6 +32,8 @@ the canonical authority spec on top of those. | **Decision Quality Roadmap** | [`docs/plans/2026-07-17-decision-quality-roadmap.md`](docs/plans/2026-07-17-decision-quality-roadmap.md) | H0-H4 prove-it roadmap: Elite correctness, randomized ablations, source-grounded Decision Briefs, buyer pull, outcome learning, old-backlog disposition, demand gates, and kill criteria. | | **H1 Evaluation Design** | [`docs/plans/2026-07-17-h1-budget-matched-evaluation-design.md`](docs/plans/2026-07-17-h1-budget-matched-evaluation-design.md) | DSE-708 experimental boundary, six frozen conditions, replay, blinding, and failure-inclusive analysis. | | **H1 Evaluation Plan** | [`docs/plans/2026-07-17-h1-budget-matched-evaluation.md`](docs/plans/2026-07-17-h1-budget-matched-evaluation.md) | TDD delivery plan for the offline, budget-matched evaluation substrate. | +| **H1 Method Hardening** | [`docs/plans/2026-07-17-h1-method-hardening-design.md`](docs/plans/2026-07-17-h1-method-hardening-design.md) | Paid-study provenance, task-clustered inference, grader controls, confirmatory refusal boundaries, and pilot design. | +| **H1 Pilot Pack** | [`studies/elite_pilot_v1/README.md`](studies/elite_pilot_v1/README.md) | Balanced 24-task exploratory pack, grader-only keys, frozen pilot protocol, and confirmatory preregistration template. | --- diff --git a/README.md b/README.md index f275334..a7e3c86 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,8 @@ is all known providers. The source branch also exposes an H1 evidence harness. It makes no decision-quality claim and never calls providers: `run` only validates an existing replay artifact against its frozen manifest. Grader output and the restricted blind map must remain separate. +The frozen 24-task exploratory pack and its no-promotion boundary are documented in +[`studies/elite_pilot_v1/README.md`](studies/elite_pilot_v1/README.md). ```bash conclave eval plan tasks.json manifest.json --study-id pilot --replicates 2 --seed 19 --max-output-tokens 1200 From 730e24636998fc7738f071de94b397ec66225d4c Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:46:31 -0400 Subject: [PATCH 09/13] docs(evals): reclassify exposed pilot pack as synthetic QA --- DOCUMENTATION_INDEX.md | 4 +- README.md | 8 ++-- .../2026-07-17-h1-method-hardening-design.md | 24 ++++++++--- docs/plans/2026-07-17-h1-method-hardening.md | 17 +++++--- studies/elite_pilot_v1/README.md | 38 ---------------- studies/elite_qa_v1/README.md | 43 +++++++++++++++++++ .../confirmatory_preregistration.md | 20 ++++++--- .../grader_keys.json | 0 .../public_tasks.json | 0 .../qa_protocol.md} | 34 ++++++++------- .../{test_pilot_pack.py => test_qa_pack.py} | 21 +++++---- 11 files changed, 125 insertions(+), 84 deletions(-) delete mode 100644 studies/elite_pilot_v1/README.md create mode 100644 studies/elite_qa_v1/README.md rename studies/{elite_pilot_v1 => elite_qa_v1}/confirmatory_preregistration.md (82%) rename studies/{elite_pilot_v1 => elite_qa_v1}/grader_keys.json (100%) rename studies/{elite_pilot_v1 => elite_qa_v1}/public_tasks.json (100%) rename studies/{elite_pilot_v1/pilot_protocol.md => elite_qa_v1/qa_protocol.md} (72%) rename tests/evals/{test_pilot_pack.py => test_qa_pack.py} (80%) diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index 52fab06..0d0d128 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -32,8 +32,8 @@ the canonical authority spec on top of those. | **Decision Quality Roadmap** | [`docs/plans/2026-07-17-decision-quality-roadmap.md`](docs/plans/2026-07-17-decision-quality-roadmap.md) | H0-H4 prove-it roadmap: Elite correctness, randomized ablations, source-grounded Decision Briefs, buyer pull, outcome learning, old-backlog disposition, demand gates, and kill criteria. | | **H1 Evaluation Design** | [`docs/plans/2026-07-17-h1-budget-matched-evaluation-design.md`](docs/plans/2026-07-17-h1-budget-matched-evaluation-design.md) | DSE-708 experimental boundary, six frozen conditions, replay, blinding, and failure-inclusive analysis. | | **H1 Evaluation Plan** | [`docs/plans/2026-07-17-h1-budget-matched-evaluation.md`](docs/plans/2026-07-17-h1-budget-matched-evaluation.md) | TDD delivery plan for the offline, budget-matched evaluation substrate. | -| **H1 Method Hardening** | [`docs/plans/2026-07-17-h1-method-hardening-design.md`](docs/plans/2026-07-17-h1-method-hardening-design.md) | Paid-study provenance, task-clustered inference, grader controls, confirmatory refusal boundaries, and pilot design. | -| **H1 Pilot Pack** | [`studies/elite_pilot_v1/README.md`](studies/elite_pilot_v1/README.md) | Balanced 24-task exploratory pack, grader-only keys, frozen pilot protocol, and confirmatory preregistration template. | +| **H1 Method Hardening** | [`docs/plans/2026-07-17-h1-method-hardening-design.md`](docs/plans/2026-07-17-h1-method-hardening-design.md) | Paid-study provenance, task-clustered inference, grader controls, confirmatory refusal boundaries, and QA design. | +| **H1 Synthetic QA Pack** | [`studies/elite_qa_v1/README.md`](studies/elite_qa_v1/README.md) | Balanced 24-task open-book harness fixture, QA protocol, and confirmatory preregistration template. | --- diff --git a/README.md b/README.md index a7e3c86..269c075 100644 --- a/README.md +++ b/README.md @@ -154,11 +154,13 @@ is all known providers. The source branch also exposes an H1 evidence harness. It makes no decision-quality claim and never calls providers: `run` only validates an existing replay artifact against its frozen manifest. Grader output and the restricted blind map must remain separate. -The frozen 24-task exploratory pack and its no-promotion boundary are documented in -[`studies/elite_pilot_v1/README.md`](studies/elite_pilot_v1/README.md). +The open-book 24-task synthetic QA pack and its no-promotion boundary are documented in +[`studies/elite_qa_v1/README.md`](studies/elite_qa_v1/README.md). Its committed answer keys +are test fixtures, not access-controlled grader material, so this pack cannot be used for a +paid or confirmatory study. ```bash -conclave eval plan tasks.json manifest.json --study-id pilot --replicates 2 --seed 19 --max-output-tokens 1200 +conclave eval plan tasks.json manifest.json --study-id qa --replicates 2 --seed 19 --max-output-tokens 1200 conclave eval run manifest.json validated-run.json --replay-artifact recorded-run.json conclave eval blind validated-run.json grader.json restricted-map.json --seed 23 conclave eval report manifest.json validated-run.json judgments.json report.json report.md --bootstrap-seed 29 diff --git a/docs/plans/2026-07-17-h1-method-hardening-design.md b/docs/plans/2026-07-17-h1-method-hardening-design.md index 25e0306..e4e5012 100644 --- a/docs/plans/2026-07-17-h1-method-hardening-design.md +++ b/docs/plans/2026-07-17-h1-method-hardening-design.md @@ -2,11 +2,14 @@ **Linear:** DSE-708 **Dependency:** PR #52 at `4365c28` -**Gate:** Required before any paid pilot call +**Gate:** Required before any paid study call ## Decision -Keep the existing H1 substrate as the execution primitive, then add a frozen study-method contract and a balanced 24-task exploratory pack. The pilot estimates variance, grader reliability, leakage, latency, and cost; it cannot produce a product GO or KILL. +Keep the existing H1 substrate as the execution primitive, then add a frozen study-method +contract and a balanced 24-task open-book synthetic QA pack. The committed pack validates +mechanics only; it cannot estimate unbiased effects or produce a product GO or KILL. A paid +pilot requires separately access-controlled grader keys and a hash frozen before execution. ## Frozen study contract @@ -26,15 +29,24 @@ Non-success records bypass the human queue and are automatic failures. Successfu ## Analysis and gates -Pilot reporting uses failure-inclusive Wilson rates, task-clustered paired bootstrap differences with roster/replicate averaging, raw agreement, prevalence, kappa, adjudication rate, severe errors, reviewer effort, token/cost distributions, latency distributions, deviations, and leakage guesses. +Study reporting uses failure-inclusive Wilson rates, task-clustered paired bootstrap differences with roster/replicate averaging, raw agreement, prevalence, kappa, adjudication rate, severe errors, reviewer effort, token/cost distributions, latency distributions, deviations, and leakage guesses. -Reliability is undefined—not perfect—when expected agreement is one. The pilot is method-ready only with at least 95% double grading, raw agreement at least 80%, kappa at least 0.60 overall and no family below 0.50, adjudication at most 20%, and no material leakage. Otherwise redesign. +Reliability is undefined—not perfect—when expected agreement is one. A future paid pilot is +method-ready only with at least 95% double grading, raw agreement at least 80%, kappa at least +0.60 overall and no family below 0.50, adjudication at most 20%, and no material leakage. +Otherwise redesign. The committed open-book QA pack cannot satisfy or measure this gate. Confirmation freezes one strongest baseline. GO requires a point gain of at least 10 percentage points with the paired 95% interval above zero, severe-error noninferiority within +2 points, readiness noninferiority within +5 points, reviewer-effort and latency gates, positive direction in every family and roster, and failure-inclusive validity. Simpler-baseline equivalence, harm, weak reliability, or post-hoc changes trigger redesign or kill per DSE-708. -## Pilot pack +## Synthetic QA pack -Use 24 synthetic, current-fact-free tasks: 12 operational-execution and 12 organizational-stewardship, balanced across six subfamilies, three difficulty tiers, and `ready`/`not_ready`/`indeterminate`. Public packets and grader-only keys are separate. Confirmatory tasks are independently authored, access-restricted, and screened for sentence, numeric, structural, and semantic overlap. +Use 24 synthetic, current-fact-free tasks: 12 operational-execution and 12 +organizational-stewardship, balanced across six subfamilies, three difficulty tiers, and +`ready`/`not_ready`/`indeterminate`. Public packets and committed fixture keys are separate +files, but not separate security domains. Paid-study keys must be provisioned outside the +repository under access control and frozen by hash. Confirmatory tasks are independently +authored, access-restricted, and screened for sentence, numeric, structural, and semantic +overlap. ## Non-goals diff --git a/docs/plans/2026-07-17-h1-method-hardening.md b/docs/plans/2026-07-17-h1-method-hardening.md index e470dd0..8fe63af 100644 --- a/docs/plans/2026-07-17-h1-method-hardening.md +++ b/docs/plans/2026-07-17-h1-method-hardening.md @@ -20,10 +20,14 @@ 2. Extend `scoring.py`, `reporting.py`, and CLI artifact validation. Archive every gate input and seed. 3. Re-run focused tests and commit. -## Phase 4 — Pilot pack - -1. Add `studies/elite_pilot_v1/public_tasks.json`, embargoed-development `grader_keys.json`, `pilot_protocol.md`, `confirmatory_preregistration.md`, and `README.md`. -2. Add `tests/evals/test_pilot_pack.py`: exactly 24 unique tasks, 12/12 families, 4 per subfamily, balanced tier/readiness, matching nonempty keys, packet citation IDs, no grader material in public records, no duplicates, and documented leakage/holdout rules. +## Phase 4 — Synthetic QA pack + +1. Add `studies/elite_qa_v1/public_tasks.json`, committed fixture `grader_keys.json`, + `qa_protocol.md`, `confirmatory_preregistration.md`, and `README.md`. Classify the pack as + open-book harness QA, not a paid pilot. +2. Add `tests/evals/test_qa_pack.py`: exactly 24 unique tasks, 12/12 families, 4 per + subfamily, balanced tier/readiness, matching nonempty fixture keys, packet citation IDs, no + grader material in public records, no duplicates, and documented leakage/holdout rules. 3. Generate and validate an offline manifest only; do not create model outputs. 4. Re-run focused tests and commit. @@ -39,4 +43,7 @@ git diff --check gitleaks git --redact ``` -Push, open a draft PR stacked on #52, obtain independent methods/code review, and attach evidence to DSE-708. Paid pilot execution remains blocked until PR #51 is merged/pinned and Ernest approves a hard spend ceiling. +Push, open a draft PR stacked on #52, obtain independent methods/code review, and attach +evidence to DSE-708. Paid pilot execution remains blocked until PR #51 is merged/pinned, +Ernest approves a hard spend ceiling, and separately access-controlled grader keys are frozen +by hash before execution. diff --git a/studies/elite_pilot_v1/README.md b/studies/elite_pilot_v1/README.md deleted file mode 100644 index 075d180..0000000 --- a/studies/elite_pilot_v1/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Elite Pilot v1 - -This directory is a **synthetic exploratory** evaluation pack for debugging the Conclave H1 -study method. It is not a production benchmark, a confirmatory study, or evidence that Elite -improves decisions. Its results **must not support product-quality claims**. - -## Contents - -- `public_tasks.json` contains the 24 synthetic tasks visible to every experimental condition. -- `grader_keys.json` contains expected decisions, critical errors, and rubric anchors. It is - grader-only and must never be loaded by a protocol runner or included in a model prompt. -- `pilot_protocol.md` freezes the exploratory execution and grading rules. -- `confirmatory_preregistration.md` is a gated template for a later, independently authored - held-out study. It is not an active preregistration. - -The pack has two prespecified macro-families with 12 tasks each, six subfamilies with four -tasks each, and balanced tiers and expected readiness outcomes: - -| Axis | Distribution | -| --- | --- | -| Macro-family | 12 `operational_execution`; 12 `organizational_stewardship` | -| Subfamily | 4 each: procurement, release operations, product experiments, capital allocation, governance, staffing | -| Difficulty | 8 each: `tier_1`, `tier_2`, `tier_3` | -| Keyed readiness | 8 each: `ready`, `not_ready`, `indeterminate` | - -All facts, organizations, people, products, and prices are fictional. The tasks require no -current knowledge and prohibit external assumptions. Public packets and grader keys use the -immutable `conclave_eval_v1` contracts and can be loaded with -`conclave.evals.dataset.load_public_tasks` and `load_grader_keys`. - -## Frozen pilot-only boundary - -Pilot v1 may debug task clarity, rubric wording, grader agreement, execution reliability, and -confirmatory sample-size assumptions. It may not establish superiority, authorize product -claims, tune a confirmatory holdout, or influence real decisions. Any material task, key, -prompt, rubric, or protocol change creates a new version and new hashes; it must not silently -replace this pack. - diff --git a/studies/elite_qa_v1/README.md b/studies/elite_qa_v1/README.md new file mode 100644 index 0000000..d4b854d --- /dev/null +++ b/studies/elite_qa_v1/README.md @@ -0,0 +1,43 @@ +# Elite Synthetic QA v1 + +This directory is an **open-book synthetic harness QA** pack for debugging the Conclave H1 +study method. It is not a paid pilot, production benchmark, confirmatory study, or evidence +that Elite improves decisions. Its results **must not support product-quality claims**. + +## Contents + +- `public_tasks.json` contains the 24 synthetic tasks visible to every experimental condition. +- `grader_keys.json` contains expected decisions, critical errors, and rubric anchors. Because + it is committed beside the tasks, it is an open-book test fixture, not access-controlled + grader material. +- `qa_protocol.md` freezes the synthetic QA execution and grading rules. +- `confirmatory_preregistration.md` is a gated template for a later, independently authored + held-out study. It is not an active preregistration. + +The pack has two prespecified macro-families with 12 tasks each, six subfamilies with four +tasks each, and balanced tiers and expected readiness outcomes: + +| Axis | Distribution | +| --- | --- | +| Macro-family | 12 `operational_execution`; 12 `organizational_stewardship` | +| Subfamily | 4 each: procurement, release operations, product experiments, capital allocation, governance, staffing | +| Difficulty | 8 each: `tier_1`, `tier_2`, `tier_3` | +| Keyed readiness | 8 each: `ready`, `not_ready`, `indeterminate` | + +All facts, organizations, people, products, and prices are fictional. The tasks require no +current knowledge and prohibit external assumptions. Public packets and grader keys use the +immutable `conclave_eval_v1` contracts and can be loaded with +`conclave.evals.dataset.load_public_tasks` and `load_grader_keys`. + +## Frozen QA-only boundary + +QA v1 may debug schemas, task clarity, rubric wording, grading workflows, execution +reliability, and report generation. It may not estimate unbiased model quality, establish +superiority, authorize product claims, tune a confirmatory holdout, or influence real +decisions. Any material task, key, prompt, rubric, or protocol change creates a new version +and new hashes; it must not silently replace this pack. + +A paid execution requires a separately provisioned key artifact that model and runner +identities cannot read, an access-controlled grading workflow, and a frozen cryptographic +hash recorded before execution. The committed `grader_keys.json` must never be relabeled or +copied as that paid-study key artifact. diff --git a/studies/elite_pilot_v1/confirmatory_preregistration.md b/studies/elite_qa_v1/confirmatory_preregistration.md similarity index 82% rename from studies/elite_pilot_v1/confirmatory_preregistration.md rename to studies/elite_qa_v1/confirmatory_preregistration.md index af6b395..86c3c35 100644 --- a/studies/elite_pilot_v1/confirmatory_preregistration.md +++ b/studies/elite_qa_v1/confirmatory_preregistration.md @@ -12,7 +12,8 @@ - Primary endpoint: failure-inclusive critical-error-free decision rate: `[confirm]` - Primary comparison and multiplicity treatment: `[required]` - Minimum practically important effect: `[required]` -- Power, alpha, sample-size method, and assumptions justified from pilot variance: `[required]` +- Power, alpha, sample-size method, and assumptions justified independently of this open-book + QA pack: `[required]` - Two prespecified macro-families and minimum family-specific sample sizes: `[required]` - Conditions, prompts, roster, provider/model versions, budgets, replicates, and seed: `[required]` - Inclusion, failure, retry, exclusion, stopping, and missing-data rules: `[required]` @@ -21,10 +22,10 @@ ## Independent holdout construction -The confirmatory corpus must be authored as a separate project by people who cannot see pilot -outputs while authoring keys. It must use **new scenario archetypes**, entities, numbers, +The confirmatory corpus must be authored as a separate project by people who cannot see QA or +paid-pilot outputs while authoring keys. It must use **new scenario archetypes**, entities, numbers, packet prose, constraint interactions, and minority-view traps. Simple **parameter swaps** or -paraphrases of pilot tasks are prohibited. Pilot task or answer text may not enter examples, +paraphrases of QA tasks are prohibited. QA task or answer text may not enter examples, few-shot context, prompt tuning, grader training, or rubric demonstrations. Before acceptance, run exact-ID and exact-text checks, an **eight-token** contiguous-overlap @@ -35,7 +36,12 @@ task; do not repair it by editing only its answer key. Public tasks and grader keys remain separate. The execution identity cannot read the keys. Corpus authors do not grade outputs they authored unless declared and sensitivity-tested. -Grader training uses separate calibration examples that are in neither pilot nor holdout. +Grader training uses separate calibration examples that are in neither this QA pack nor the +holdout. + +Committed repository keys are fixtures only. Any paid pilot or confirmatory run requires a +separately access-controlled key artifact, unavailable to models and runner identities, whose +cryptographic hash is frozen in the study manifest before execution. ## Freeze and unsealing gate @@ -47,8 +53,8 @@ code, primary comparison, minimum effect, sample size, stopping rules, and decis An independent reviewer must attest that: -1. the pilot and holdout are semantically distinct; -2. no model, protocol author, or runner has received grader-only material; +1. this QA pack, any paid pilot, and the holdout are semantically distinct; +2. no model, protocol author, or runner has received the access-controlled study keys; 3. budgets and evidence access are condition-matched; 4. the full failure-inclusive analysis can run from frozen artifacts; 5. provider spend and automatic-stop authority are approved; and diff --git a/studies/elite_pilot_v1/grader_keys.json b/studies/elite_qa_v1/grader_keys.json similarity index 100% rename from studies/elite_pilot_v1/grader_keys.json rename to studies/elite_qa_v1/grader_keys.json diff --git a/studies/elite_pilot_v1/public_tasks.json b/studies/elite_qa_v1/public_tasks.json similarity index 100% rename from studies/elite_pilot_v1/public_tasks.json rename to studies/elite_qa_v1/public_tasks.json diff --git a/studies/elite_pilot_v1/pilot_protocol.md b/studies/elite_qa_v1/qa_protocol.md similarity index 72% rename from studies/elite_pilot_v1/pilot_protocol.md rename to studies/elite_qa_v1/qa_protocol.md index 982cdec..00cdde4 100644 --- a/studies/elite_pilot_v1/pilot_protocol.md +++ b/studies/elite_qa_v1/qa_protocol.md @@ -1,11 +1,11 @@ -# Elite Pilot v1 Protocol +# Elite Synthetic QA v1 Protocol ## Purpose and decision boundary -This is a frozen, synthetic, exploratory pilot. It tests whether the H1 harness, task design, -atomic rubric, blinding, and adjudication process are usable. It cannot demonstrate that a -condition is better, justify a product-quality claim, or change a real decision. The primary -pilot outcome is method readiness for a separately authored confirmatory study. +This is a frozen, open-book synthetic harness QA exercise. It tests whether the H1 schemas, +task design, atomic rubric, blinding mechanics, and adjudication process are usable. It cannot +demonstrate that a condition is better, justify a product-quality claim, estimate an unbiased +effect, or change a real decision. Its output is QA evidence only. ## Frozen corpus and prompt @@ -27,16 +27,20 @@ replicate count, seed, software revision, and replay-artifact hashes are recorde ## Execution -Only `public_tasks.json` may enter execution. `grader_keys.json` is grader-only and must be -kept outside the runner's readable inputs. Every planned cell produces exactly one record. +Only `public_tasks.json` may enter an execution-path QA test. The committed +`grader_keys.json` is an open-book fixture and must still be excluded from model prompts and +runner inputs to exercise the intended interface. This convention is not an access control. +Every planned cell produces exactly one record. Timeouts, malformed output, abstentions, incomplete output, provider errors, and missing records are non-successes; **failures remain in the denominator**. No cell may be rerun or excluded because its answer is inconvenient. Any permitted infrastructure retry must be predeclared and retain every attempt in the audit artifact. -This repository pack does not authorize paid calls. A live pilot requires a separately -approved spend ceiling, an automatic stop below that ceiling, pinned model identifiers, and a -fresh manifest. Offline replay is the default until those gates are satisfied. +This repository pack does not authorize paid calls. A paid execution requires separately +access-controlled grader keys, their cryptographic hash frozen before execution, an approved +spend ceiling, an automatic stop below that ceiling, pinned model identifiers, and a fresh +manifest. The committed fixture keys do not satisfy that gate. Offline replay is the only +valid use of this pack. ## Atomic grading @@ -67,14 +71,14 @@ minutes. No composite quality score is reported. Grader disagreements retain both raw judgments and receive a separate adjudication citing exactly the source judgments. Report Cohen's kappa, disagreement count, adjudication rate, Wilson intervals for rates, and seeded task-paired bootstrap intervals for Elite-minus-each- -baseline differences. Report macro-family and subfamily results descriptively; this pilot is -not powered for confirmatory subgroup claims. +baseline differences. Report macro-family and subfamily results descriptively; this QA pack +is neither blinded evidence nor powered for confirmatory subgroup claims. ## Frozen exclusions and changes Exclude no task or output after execution. Pre-execution rejection is allowed only for a schema-invalid or hash-mismatched entire pack, which aborts the study rather than deleting cells. Changes to a scenario, packet, key, prompt, condition, dimension, critical-error rule, -budget rule, or analysis create a new pack version. Pilot observations may improve a future -rubric and estimate variance; they may not alter, reveal, or train on confirmatory tasks. - +budget rule, or analysis create a new pack version. QA observations may improve harness +mechanics or a future rubric; they may not supply effect estimates, alter, reveal, or train on +confirmatory tasks. diff --git a/tests/evals/test_pilot_pack.py b/tests/evals/test_qa_pack.py similarity index 80% rename from tests/evals/test_pilot_pack.py rename to tests/evals/test_qa_pack.py index 0ee9c54..7b49dac 100644 --- a/tests/evals/test_pilot_pack.py +++ b/tests/evals/test_qa_pack.py @@ -6,7 +6,7 @@ from conclave.evals.dataset import load_grader_keys, load_public_tasks ROOT = Path(__file__).resolve().parents[2] -PACK = ROOT / "studies" / "elite_pilot_v1" +PACK = ROOT / "studies" / "elite_qa_v1" EXPECTED_TASK_IDS = { *(f"PRC-{index:02d}" for index in range(1, 5)), @@ -42,7 +42,7 @@ def _expected_readiness(required_facts: tuple[str, ...]) -> str: return matches[0] -def test_pilot_pack_is_complete_balanced_and_public_private_separated() -> None: +def test_qa_pack_is_complete_balanced_and_task_key_files_are_separate() -> None: tasks = load_public_tasks(PACK / "public_tasks.json") keys = load_grader_keys(PACK / "grader_keys.json") @@ -84,16 +84,21 @@ def test_pilot_pack_is_complete_balanced_and_public_private_separated() -> None: assert "critical_errors" not in public_text -def test_pilot_protocol_freezes_exploratory_boundary_and_holdout_controls() -> None: +def test_qa_protocol_freezes_open_book_boundary_and_holdout_controls() -> None: readme = (PACK / "README.md").read_text(encoding="utf-8") - protocol = (PACK / "pilot_protocol.md").read_text(encoding="utf-8") + protocol = (PACK / "qa_protocol.md").read_text(encoding="utf-8") preregistration = (PACK / "confirmatory_preregistration.md").read_text(encoding="utf-8") + normalized_protocol = " ".join(protocol.lower().split()) - assert "synthetic exploratory" in readme.lower() + assert "open-book synthetic harness qa" in readme.lower() assert "must not support product-quality claims" in readme.lower() - assert "critical-error-free decision rate" in protocol.lower() - assert "failures remain in the denominator" in protocol.lower() - assert "grader-only" in protocol.lower() + assert "committed `grader_keys.json`" in readme.lower() + assert "not access-controlled" in readme.lower() + assert "critical-error-free decision rate" in normalized_protocol + assert "failures remain in the denominator" in normalized_protocol + assert "open-book fixture" in normalized_protocol + assert "separately access-controlled grader keys" in normalized_protocol + assert "cryptographic hash frozen before execution" in normalized_protocol assert "not a completed preregistration" in preregistration.lower() assert "new scenario archetypes" in preregistration.lower() assert "parameter swaps" in preregistration.lower() From c4afbee0db6620bd5ca94a5175eb7e12cbac9948 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:53:42 -0400 Subject: [PATCH 10/13] fix(evals): enforce defensible confirmatory guardrails --- src/conclave/evals/models.py | 11 +- src/conclave/evals/scoring.py | 263 +++++++++++++++++++++++------ tests/evals/test_analysis_gates.py | 187 ++++++++++++++------ 3 files changed, 350 insertions(+), 111 deletions(-) diff --git a/src/conclave/evals/models.py b/src/conclave/evals/models.py index 72ada26..056e45f 100644 --- a/src/conclave/evals/models.py +++ b/src/conclave/evals/models.py @@ -131,10 +131,19 @@ class AnalysisGateConfig(EvalModel): severe_error_noninferiority_margin: Literal[0.02] = 0.02 readiness_noninferiority_margin: Literal[0.05] = 0.05 reviewer_effort_max_ratio: Literal[1.20] = 1.20 + reviewer_effort_statistic: Literal["ratio_of_task_medians"] = "ratio_of_task_medians" + latency_baseline: Literal["single_frontier"] = "single_frontier" p95_latency_max_ratio: Literal[3.0] = 3.0 absolute_p95_latency_seconds: float = Field(gt=0) minimum_confirmatory_tasks: int = Field(ge=2) - multiplicity_rule: Literal["holm_secondary"] = "holm_secondary" + minimum_double_grading_rate: Literal[0.95] = 0.95 + minimum_raw_agreement: Literal[0.80] = 0.80 + minimum_overall_kappa: Literal[0.60] = 0.60 + minimum_family_kappa: Literal[0.50] = 0.50 + maximum_adjudication_rate: Literal[0.20] = 0.20 + multiplicity_rule: Literal["bonferroni_simultaneous_upper_bounds"] = ( + "bonferroni_simultaneous_upper_bounds" + ) unit: Literal["task"] = "task" @model_validator(mode="after") diff --git a/src/conclave/evals/scoring.py b/src/conclave/evals/scoring.py index ca16280..a4f64ae 100644 --- a/src/conclave/evals/scoring.py +++ b/src/conclave/evals/scoring.py @@ -2,10 +2,13 @@ from __future__ import annotations +import hashlib +import json import math import random from collections import Counter, defaultdict from collections.abc import Mapping, Sequence +from statistics import median from typing import Literal from pydantic import Field, model_validator @@ -156,6 +159,9 @@ class ReliabilityMetrics(EvalModel): adjudication_rate: float = Field(ge=0.0, le=1.0) raw_agreement: float | None = Field(default=None, ge=0.0, le=1.0) positive_prevalence: float | None = Field(default=None, ge=0.0, le=1.0) + successful_runs: int = Field(default=0, ge=0) + double_graded_runs: int = Field(default=0, ge=0) + double_grading_rate: float | None = Field(default=None, ge=0.0, le=1.0) strata: tuple[ReliabilityStratum, ...] = () @@ -196,7 +202,7 @@ class StudyScoreReport(EvalModel): adjudications: tuple[AdjudicationRecord, ...] resolved_outcomes: tuple[ResolvedOutcome, ...] paid_analysis: PaidAnalysisSummary | None = None - confirmatory_inference: ConfirmatoryInference | None = None + simultaneous_upper_bounds: SimultaneousUpperBounds | None = None class RatioInterval(EvalModel): @@ -211,23 +217,29 @@ class RatioInterval(EvalModel): bootstrap_samples: int = Field(ge=1) -class ConfirmatoryInference(EvalModel): - """Predeclared inference receipts required before a confirmatory decision.""" +class SimultaneousUpperBounds(EvalModel): + """Bonferroni simultaneous one-sided bounds for the three secondary gates.""" - achieved_power: float = Field(gt=0, le=1) - secondary_p_values: dict[str, float] = Field(min_length=1) + familywise_alpha: Literal[0.05] = 0.05 + endpoint_count: Literal[3] = 3 + per_endpoint_alpha: float = Field(gt=0, lt=1) + quantile_probability: float = Field(gt=0, lt=1) + method: Literal["task_clustered_percentile_bonferroni_v1"] = ( + "task_clustered_percentile_bonferroni_v1" + ) + reviewer_effort_statistic: Literal["ratio_of_task_medians"] = "ratio_of_task_medians" + severe_error_upper_bound: float + readiness_error_upper_bound: float + reviewer_effort_ratio_upper_bound: float = Field(gt=0) + evidence_hash: Sha256Digest @model_validator(mode="after") - def validate_secondary_p_values(self) -> ConfirmatoryInference: - required = { - "severe_error_noninferiority", - "readiness_noninferiority", - "reviewer_effort", - } - if set(self.secondary_p_values) != required: - raise ValueError("secondary_p_values must exactly cover the frozen secondary gates") - if any(value < 0 or value > 1 for value in self.secondary_p_values.values()): - raise ValueError("secondary p-values must be between zero and one") + def validate_bonferroni_metadata(self) -> SimultaneousUpperBounds: + expected_alpha = self.familywise_alpha / self.endpoint_count + if not math.isclose(self.per_endpoint_alpha, expected_alpha, abs_tol=1e-15): + raise ValueError("per_endpoint_alpha must equal familywise alpha divided by endpoints") + if not math.isclose(self.quantile_probability, 1 - expected_alpha, abs_tol=1e-15): + raise ValueError("quantile_probability must equal one minus per-endpoint alpha") return self @@ -239,6 +251,7 @@ class PaidAnalysisSummary(EvalModel): readiness_error_difference: PairedDifference reviewer_effort_ratio: float = Field(gt=0) reviewer_effort_ratio_interval: RatioInterval + latency_baseline_condition_id: ConditionId p95_latency_ratio: float = Field(gt=0) elite_p95_latency_seconds: float = Field(ge=0) total_cost_usd: float = Field(ge=0) @@ -278,11 +291,22 @@ def evaluate_confirmatory_gates( raise ValueError("report preregistration hash does not match the frozen design") if report.paid_analysis is None: raise ValueError("confirmatory report is missing paid analysis") - if report.confirmatory_inference is None: - raise ValueError("confirmatory report is missing inference receipts") + if report.simultaneous_upper_bounds is None: + raise ValueError("confirmatory report is missing simultaneous upper bounds") metrics = report.paid_analysis - inference = report.confirmatory_inference + bounds = report.simultaneous_upper_bounds config = design.analysis_gates + if bounds.evidence_hash != hash_confirmatory_evidence(report): + raise ValueError("simultaneous-bound evidence hash does not match the report") + expected_per_endpoint_alpha = config.alpha / 3 + if not math.isclose(bounds.per_endpoint_alpha, expected_per_endpoint_alpha, abs_tol=1e-15): + raise ValueError("simultaneous-bound alpha does not match the frozen design") + if not math.isclose( + bounds.quantile_probability, 1 - expected_per_endpoint_alpha, abs_tol=1e-15 + ): + raise ValueError("simultaneous-bound quantile does not match the frozen design") + if bounds.reviewer_effort_statistic != config.reviewer_effort_statistic: + raise ValueError("reviewer-effort statistic does not match the frozen design") primary = metrics.primary_paired_difference if primary.baseline_condition_id != config.primary_baseline: raise ValueError("confirmatory primary baseline does not match the frozen design") @@ -296,36 +320,65 @@ def evaluate_confirmatory_gates( raise ValueError("confirmatory family directions do not match the frozen analysis set") if set(metrics.roster_directions) != expected_rosters: raise ValueError("confirmatory roster directions do not match the frozen analysis set") + family_reliability = { + item.stratum_value: item + for item in report.reliability.strata + if item.stratum_type == "family" + } + if set(family_reliability) != expected_families: + raise ValueError("confirmatory family reliability does not match the frozen analysis set") + if metrics.latency_baseline_condition_id != config.latency_baseline: + raise ValueError("confirmatory latency baseline does not match the frozen design") - ordered_p_values = sorted(inference.secondary_p_values.values()) - holm_passed = all( - value <= config.alpha / (len(ordered_p_values) - index) - for index, value in enumerate(ordered_p_values) - ) failed = [] checks = ( ("sample_size", primary.task_count >= config.minimum_confirmatory_tasks), - ("power", inference.achieved_power >= config.power), - ("holm_secondary", holm_passed), ("minimum_effect", primary.estimate >= config.minimum_effect), ("superiority_interval", primary.lower > 0), ( "severe_error_noninferiority", - metrics.severe_error_difference.upper <= config.severe_error_noninferiority_margin, + bounds.severe_error_upper_bound <= config.severe_error_noninferiority_margin, ), ( "readiness_noninferiority", - metrics.readiness_error_difference.upper <= config.readiness_noninferiority_margin, + bounds.readiness_error_upper_bound <= config.readiness_noninferiority_margin, ), ( "reviewer_effort", - metrics.reviewer_effort_ratio_interval.upper <= config.reviewer_effort_max_ratio, + bounds.reviewer_effort_ratio_upper_bound <= config.reviewer_effort_max_ratio, ), ("latency_ratio", metrics.p95_latency_ratio <= config.p95_latency_max_ratio), ( "latency_ceiling", metrics.elite_p95_latency_seconds <= config.absolute_p95_latency_seconds, ), + ("spend_ceiling", metrics.total_cost_usd <= design.approved_spend_ceiling_usd), + ( + "double_grading", + report.reliability.double_grading_rate is not None + and report.reliability.double_grading_rate >= config.minimum_double_grading_rate, + ), + ( + "raw_agreement", + report.reliability.raw_agreement is not None + and report.reliability.raw_agreement >= config.minimum_raw_agreement, + ), + ( + "overall_kappa", + report.reliability.cohen_kappa is not None + and report.reliability.cohen_kappa >= config.minimum_overall_kappa, + ), + ( + "family_kappa", + all( + item.cohen_kappa is not None and item.cohen_kappa >= config.minimum_family_kappa + for item in family_reliability.values() + ), + ), + ( + "adjudication_rate", + report.reliability.adjudication_rate <= config.maximum_adjudication_rate, + ), ("family_direction", all(value > 0 for value in metrics.family_directions.values())), ("roster_direction", all(value > 0 for value in metrics.roster_directions.values())), ) @@ -413,35 +466,94 @@ def paired_bootstrap_ratio( seed: int, samples: int, ) -> RatioInterval: - """Bootstrap a task-clustered ratio of mean repeated-measure effort.""" + """Bootstrap the preregistered ratio of median task-level reviewer effort.""" if set(numerator_by_task) != set(denominator_by_task) or not numerator_by_task: raise ValueError("ratio numerator and denominator must contain the same task IDs") task_ids = sorted(numerator_by_task) - denominator = sum(denominator_by_task[item] for item in task_ids) / len(task_ids) - if denominator <= 0: - raise ValueError("paid reviewer-effort baseline must be positive") - estimate = (sum(numerator_by_task[item] for item in task_ids) / len(task_ids)) / denominator + if any(numerator_by_task[item] <= 0 or denominator_by_task[item] <= 0 for item in task_ids): + raise ValueError("task-level reviewer effort must be strictly positive for log ratios") + numerator_median = median(numerator_by_task[item] for item in task_ids) + denominator_median = median(denominator_by_task[item] for item in task_ids) + estimate = math.exp(math.log(numerator_median / denominator_median)) rng = random.Random(seed) bootstrapped = [] for _ in range(samples): sampled = [rng.choice(task_ids) for _ in task_ids] - sampled_denominator = sum(denominator_by_task[item] for item in sampled) / len(sampled) - if sampled_denominator <= 0: - raise ValueError("reviewer-effort bootstrap encountered a zero baseline") - sampled_numerator = sum(numerator_by_task[item] for item in sampled) / len(sampled) - bootstrapped.append(sampled_numerator / sampled_denominator) + sampled_numerator = median(numerator_by_task[item] for item in sampled) + sampled_denominator = median(denominator_by_task[item] for item in sampled) + bootstrapped.append(math.exp(math.log(sampled_numerator / sampled_denominator))) bootstrapped.sort() return RatioInterval( estimate=estimate, - lower=_quantile(bootstrapped, 0.05), - upper=_quantile(bootstrapped, 0.95), + lower=_quantile(bootstrapped, 0.025), + upper=_quantile(bootstrapped, 0.975), task_count=len(task_ids), bootstrap_seed=seed, bootstrap_samples=samples, ) +def _bootstrap_difference_upper_bound( + numerator_by_task: Mapping[str, float], + denominator_by_task: Mapping[str, float], + *, + quantile_probability: float, + seed: int, + samples: int, +) -> float: + """Return an approximate task-clustered percentile upper bound.""" + + task_ids = sorted(numerator_by_task) + if set(task_ids) != set(denominator_by_task) or not task_ids: + raise ValueError("bootstrap inputs must contain the same task IDs") + rng = random.Random(seed) + bootstrapped = [] + for _ in range(samples): + sampled = [rng.choice(task_ids) for _ in task_ids] + bootstrapped.append( + sum(numerator_by_task[item] - denominator_by_task[item] for item in sampled) + / len(sampled) + ) + return _quantile(sorted(bootstrapped), quantile_probability) + + +def _bootstrap_median_ratio_upper_bound( + numerator_by_task: Mapping[str, float], + denominator_by_task: Mapping[str, float], + *, + quantile_probability: float, + seed: int, + samples: int, +) -> float: + """Return an approximate task-clustered percentile upper bound for effort ratio.""" + + task_ids = sorted(numerator_by_task) + if set(task_ids) != set(denominator_by_task) or not task_ids: + raise ValueError("bootstrap inputs must contain the same task IDs") + if any(numerator_by_task[item] <= 0 or denominator_by_task[item] <= 0 for item in task_ids): + raise ValueError("task-level reviewer effort must be strictly positive for log ratios") + rng = random.Random(seed) + bootstrapped = [] + for _ in range(samples): + sampled = [rng.choice(task_ids) for _ in task_ids] + sampled_numerator = median(numerator_by_task[item] for item in sampled) + sampled_denominator = median(denominator_by_task[item] for item in sampled) + bootstrapped.append(math.exp(math.log(sampled_numerator / sampled_denominator))) + return _quantile(sorted(bootstrapped), quantile_probability) + + +def hash_confirmatory_evidence(report: StudyScoreReport) -> str: + """Hash every report field from which the confirmatory receipt was derived.""" + + canonical = json.dumps( + report.model_dump(mode="json", exclude={"simultaneous_upper_bounds"}), + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" + + def cohen_kappa(first: Sequence[bool], second: Sequence[bool]) -> float | None: """Return Cohen's kappa for aligned binary ratings, or ``None`` if undefined.""" @@ -479,15 +591,13 @@ def _paired_values( judgments_by_run: Mapping[str, Sequence[GraderJudgment]], run_ids: Sequence[str] ) -> tuple[tuple[str, str] | None, list[tuple[bool, bool]]]: graders = sorted({item.grader_id for values in judgments_by_run.values() for item in values}) - if len(graders) != 2: - return None, [] - first, second = graders + grader_pair = (graders[0], graders[1]) if len(graders) == 2 else None pairs = [] for run_id in sorted(run_ids): - by_grader = {item.grader_id: item.critical_error_free for item in judgments_by_run[run_id]} - if first in by_grader and second in by_grader: - pairs.append((by_grader[first], by_grader[second])) - return (first, second), pairs + ordered = sorted(judgments_by_run.get(run_id, ()), key=lambda item: item.grader_id) + if len(ordered) == 2 and ordered[0].grader_id != ordered[1].grader_id: + pairs.append((ordered[0].critical_error_free, ordered[1].critical_error_free)) + return grader_pair, pairs def _agreement_statistics( @@ -506,7 +616,14 @@ def _reliability( judgments_by_run: Mapping[str, Sequence[GraderJudgment]], adjudicated_runs: set[str], manifest: StudyManifest, + study_run: StudyRun, ) -> ReliabilityMetrics: + analysis_ids = {run.planned_run_id for run in analysis_planned_runs(manifest)} + judgments_by_run = { + run_id: values for run_id, values in judgments_by_run.items() if run_id in analysis_ids + } + outcomes = {record.planned_run_id: record.outcome for record in study_run.records} + successful_runs = sum(outcomes[run_id] == "success" for run_id in analysis_ids) disagreements = sum( len({item.critical_error_free for item in values}) > 1 for values in judgments_by_run.values() @@ -564,6 +681,9 @@ def _reliability( adjudication_rate=rate, raw_agreement=raw_agreement, positive_prevalence=prevalence, + successful_runs=successful_runs, + double_graded_runs=len(pairs), + double_grading_rate=(len(pairs) / successful_runs if successful_runs else None), strata=tuple(strata), ) @@ -649,7 +769,6 @@ def score_study( blind_map: BlindMap | None = None, bootstrap_seed: int, bootstrap_samples: int, - confirmatory_inference: ConfirmatoryInference | None = None, ) -> StudyScoreReport: """Score one frozen study without dropping failures or modifying raw judgments.""" @@ -660,8 +779,6 @@ def score_study( or bootstrap_samples != manifest.frozen_design.bootstrap.samples ): raise ValueError("paid scoring must use the frozen bootstrap seed and sample count") - if confirmatory_inference is not None and manifest.evidence_classification != "confirmatory": - raise ValueError("confirmatory inference can only be attached to confirmatory evidence") validate_run_records(manifest, study_run.records) planned_by_id = {run.planned_run_id: run for run in manifest.planned_runs} run_by_id = {record.planned_run_id: record for record in study_run.records} @@ -817,7 +934,6 @@ def task_means(cell_values: Mapping[str, float]) -> dict[ConditionId, dict[str, effort_cells = { run.planned_run_id: ( sum(item.reviewer_seconds or 0 for item in judgments_by_run[run.planned_run_id]) - / len(judgments_by_run[run.planned_run_id]) if judgments_by_run[run.planned_run_id] else 0.0 ) @@ -848,15 +964,16 @@ def task_means(cell_values: Mapping[str, float]) -> dict[ConditionId, dict[str, seed=frozen_seed, samples=frozen_samples, ) + latency_baseline = design.analysis_gates.latency_baseline latencies = { condition: sorted( (run_by_id[run.planned_run_id].latency_ms or 0.0) / 1000 for run in analysis_runs if run.condition_id == condition ) - for condition in (baseline, "elite_full") + for condition in (latency_baseline, "elite_full") } - baseline_p95 = _quantile(latencies[baseline], 0.95) + baseline_p95 = _quantile(latencies[latency_baseline], 0.95) elite_p95 = _quantile(latencies["elite_full"], 0.95) if baseline_p95 <= 0: raise ValueError("paid latency baseline must be positive") @@ -898,6 +1015,7 @@ def selected(run, name): readiness_error_difference=readiness_difference, reviewer_effort_ratio=effort_interval.estimate, reviewer_effort_ratio_interval=effort_interval, + latency_baseline_condition_id=latency_baseline, p95_latency_ratio=elite_p95 / baseline_p95, elite_p95_latency_seconds=elite_p95, total_cost_usd=sum(record.cost_usd for record in study_run.records), @@ -909,7 +1027,7 @@ def selected(run, name): family_directions=directional_effects("family"), roster_directions=directional_effects("roster"), ) - return StudyScoreReport( + report = StudyScoreReport( study_id=manifest.study_id, evidence_classification=manifest.evidence_classification, frozen_design_hash=manifest.frozen_design_hash, @@ -934,10 +1052,43 @@ def selected(run, name): ), condition_metrics=tuple(condition_metrics), paired_differences=paired, - reliability=_reliability(judgments_by_run, set(adjudication_by_run), manifest), + reliability=_reliability(judgments_by_run, set(adjudication_by_run), manifest, study_run), raw_judgments=tuple(raw_judgments), adjudications=tuple(adjudications), resolved_outcomes=tuple(resolved), paid_analysis=paid_analysis, - confirmatory_inference=confirmatory_inference, ) + if manifest.evidence_classification != "confirmatory": + return report + assert manifest.frozen_design is not None + design = manifest.frozen_design + config = design.analysis_gates + per_endpoint_alpha = config.alpha / 3 + quantile_probability = 1 - per_endpoint_alpha + bounds = SimultaneousUpperBounds( + per_endpoint_alpha=per_endpoint_alpha, + quantile_probability=quantile_probability, + severe_error_upper_bound=_bootstrap_difference_upper_bound( + severe_means["elite_full"], + severe_means[baseline], + quantile_probability=quantile_probability, + seed=design.bootstrap.seed, + samples=design.bootstrap.samples, + ), + readiness_error_upper_bound=_bootstrap_difference_upper_bound( + readiness_means["elite_full"], + readiness_means[baseline], + quantile_probability=quantile_probability, + seed=design.bootstrap.seed, + samples=design.bootstrap.samples, + ), + reviewer_effort_ratio_upper_bound=_bootstrap_median_ratio_upper_bound( + effort_means["elite_full"], + effort_means[baseline], + quantile_probability=quantile_probability, + seed=design.bootstrap.seed, + samples=design.bootstrap.samples, + ), + evidence_hash=hash_confirmatory_evidence(report), + ) + return report.model_copy(update={"simultaneous_upper_bounds": bounds}) diff --git a/tests/evals/test_analysis_gates.py b/tests/evals/test_analysis_gates.py index 17318d3..96069ab 100644 --- a/tests/evals/test_analysis_gates.py +++ b/tests/evals/test_analysis_gates.py @@ -6,9 +6,10 @@ from conclave.evals.models import AnalysisGateConfig, RunRecord, StudyRun from conclave.evals.protocols import CONDITION_IDS, build_study_manifest from conclave.evals.scoring import ( - ConfirmatoryInference, analysis_planned_runs, evaluate_confirmatory_gates, + hash_confirmatory_evidence, + paired_bootstrap_ratio, score_study, ) from tests.evals.test_method_grading import _judgment @@ -34,27 +35,21 @@ def test_analysis_gate_config_freezes_canonical_dse_708_thresholds() -> None: assert config.severe_error_noninferiority_margin == 0.02 assert config.readiness_noninferiority_margin == 0.05 assert config.reviewer_effort_max_ratio == 1.20 + assert config.reviewer_effort_statistic == "ratio_of_task_medians" + assert config.latency_baseline == "single_frontier" assert config.p95_latency_max_ratio == 3.0 + assert config.minimum_double_grading_rate == 0.95 + assert config.minimum_raw_agreement == 0.80 + assert config.minimum_overall_kappa == 0.60 + assert config.minimum_family_kappa == 0.50 + assert config.maximum_adjudication_rate == 0.20 assert config.minimum_confirmatory_tasks == 2 - assert config.multiplicity_rule == "holm_secondary" + assert config.multiplicity_rule == "bonferroni_simultaneous_upper_bounds" assert config.unit == "task" with pytest.raises(ValidationError, match="primary_baseline"): _config(primary_baseline="elite_full") -def _passing_inference(**updates): - values = { - "achieved_power": 0.80, - "secondary_p_values": { - "severe_error_noninferiority": 0.01, - "readiness_noninferiority": 0.01, - "reviewer_effort": 0.01, - }, - } - values.update(updates) - return ConfirmatoryInference(**values) - - def _confirmatory_report(*, minimum_tasks: int = 2): design = _design( evidence_classification="confirmatory", @@ -76,6 +71,24 @@ def _confirmatory_report(*, minimum_tasks: int = 2): output_token_budgets=dict.fromkeys(CONDITION_IDS, 100), frozen_design=design, ) + study_run = _confirmatory_report_run(manifest) + records = study_run.records + judgments = tuple( + _judgment(record.planned_run_id, grader, value=planned.condition_id != "self_refine") + for planned, record in zip(manifest.planned_runs, records, strict=True) + for grader in ("grader-a", "grader-b") + ) + report = score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments, + bootstrap_seed=design.bootstrap.seed, + bootstrap_samples=design.bootstrap.samples, + ) + return manifest, report + + +def _confirmatory_report_run(manifest): records = tuple( RunRecord( planned_run_id=planned.planned_run_id, @@ -96,23 +109,22 @@ def _confirmatory_report(*, minimum_tasks: int = 2): total_latency_ms=sum(record.latency_ms for record in records), total_cost_usd=sum(record.cost_usd for record in records), ) - judgments = tuple( - _judgment(record.planned_run_id, grader, value=planned.condition_id != "self_refine") - for planned, record in zip(manifest.planned_runs, records, strict=True) - for grader in ("grader-a", "grader-b") - ) - report = score_study( - manifest=manifest, - study_run=study_run, - raw_judgments=judgments, - bootstrap_seed=design.bootstrap.seed, - bootstrap_samples=design.bootstrap.samples, - confirmatory_inference=_passing_inference(), + return study_run + + +def _rebind_evidence(report): + bounds = report.simultaneous_upper_bounds + assert bounds is not None + return report.model_copy( + update={ + "simultaneous_upper_bounds": bounds.model_copy( + update={"evidence_hash": hash_confirmatory_evidence(report)} + ) + } ) - return manifest, report -def test_confirmatory_gate_is_bound_to_manifest_report_power_and_holm() -> None: +def test_confirmatory_gate_is_bound_to_manifest_report_and_simultaneous_bounds() -> None: manifest, report = _confirmatory_report() result = evaluate_confirmatory_gates(manifest=manifest, report=report) assert result.status == "GO" @@ -125,26 +137,20 @@ def test_confirmatory_gate_is_bound_to_manifest_report_power_and_holm() -> None: ).failed_gates ) - weak_power = report.model_copy( - update={"confirmatory_inference": _passing_inference(achieved_power=0.79)} - ) - assert "power" in evaluate_confirmatory_gates(manifest=manifest, report=weak_power).failed_gates - - holm_failure = report.model_copy( + assert report.simultaneous_upper_bounds is not None + assert report.simultaneous_upper_bounds.method == "task_clustered_percentile_bonferroni_v1" + assert report.simultaneous_upper_bounds.endpoint_count == 3 + assert report.simultaneous_upper_bounds.per_endpoint_alpha == pytest.approx(0.05 / 3) + assert report.simultaneous_upper_bounds.quantile_probability == pytest.approx(1 - 0.05 / 3) + tampered = report.model_copy( update={ - "confirmatory_inference": _passing_inference( - secondary_p_values={ - "severe_error_noninferiority": 0.01, - "readiness_noninferiority": 0.03, - "reviewer_effort": 0.04, - } + "simultaneous_upper_bounds": report.simultaneous_upper_bounds.model_copy( + update={"evidence_hash": "sha256:" + "f" * 64} ) } ) - assert ( - "holm_secondary" - in evaluate_confirmatory_gates(manifest=manifest, report=holm_failure).failed_gates - ) + with pytest.raises(ValueError, match="evidence hash"): + evaluate_confirmatory_gates(manifest=manifest, report=tampered) def test_confirmatory_gate_uses_every_derived_report_boundary() -> None: @@ -167,11 +173,7 @@ def test_confirmatory_gate_uses_every_derived_report_boundary() -> None: } ), "severe_error_noninferiority": paid.model_copy( - update={ - "severe_error_difference": paid.severe_error_difference.model_copy( - update={"upper": 0.021} - ) - } + update={"severe_error_difference": paid.severe_error_difference} ), "readiness_noninferiority": paid.model_copy( update={ @@ -189,6 +191,7 @@ def test_confirmatory_gate_uses_every_derived_report_boundary() -> None: ), "latency_ratio": paid.model_copy(update={"p95_latency_ratio": 3.001}), "latency_ceiling": paid.model_copy(update={"elite_p95_latency_seconds": 180.1}), + "spend_ceiling": paid.model_copy(update={"total_cost_usd": 251.0}), "family_direction": paid.model_copy( update={"family_directions": {"operational": 0.0, "stewardship": 1.0}} ), @@ -197,14 +200,31 @@ def test_confirmatory_gate_uses_every_derived_report_boundary() -> None: ), } for expected_gate, changed in cases.items(): + changed_report = report.model_copy(update={"paid_analysis": changed}) + if expected_gate in { + "severe_error_noninferiority", + "readiness_noninferiority", + "reviewer_effort", + }: + bounds = changed_report.simultaneous_upper_bounds + assert bounds is not None + field = { + "severe_error_noninferiority": "severe_error_upper_bound", + "readiness_noninferiority": "readiness_error_upper_bound", + "reviewer_effort": "reviewer_effort_ratio_upper_bound", + }[expected_gate] + changed_report = changed_report.model_copy( + update={"simultaneous_upper_bounds": bounds.model_copy(update={field: 2.0})} + ) + changed_report = _rebind_evidence(changed_report) result = evaluate_confirmatory_gates( manifest=manifest, - report=report.model_copy(update={"paid_analysis": changed}), + report=changed_report, ) assert expected_gate in result.failed_gates -def test_confirmatory_gate_refuses_exploratory_missing_inference_or_provenance_drift() -> None: +def test_confirmatory_gate_refuses_exploratory_missing_bounds_or_provenance_drift() -> None: manifest, report = _confirmatory_report() with pytest.raises(ValueError, match="confirmatory"): evaluate_confirmatory_gates( @@ -213,9 +233,10 @@ def test_confirmatory_gate_refuses_exploratory_missing_inference_or_provenance_d ), report=report, ) - with pytest.raises(ValueError, match="inference"): + with pytest.raises(ValueError, match="simultaneous"): evaluate_confirmatory_gates( - manifest=manifest, report=report.model_copy(update={"confirmatory_inference": None}) + manifest=manifest, + report=report.model_copy(update={"simultaneous_upper_bounds": None}), ) with pytest.raises(ValueError, match="frozen design hash"): evaluate_confirmatory_gates( @@ -253,3 +274,61 @@ def test_task_exclusion_refuses_to_empty_a_frozen_family() -> None: with pytest.raises(ValueError, match="family operational"): analysis_planned_runs(manifest) + + +def test_reviewer_effort_ratio_uses_task_clustered_median_ratio_with_95_percent_interval() -> None: + result = paired_bootstrap_ratio( + {"a": 1.0, "b": 1.0, "c": 100.0}, + {"a": 1.0, "b": 1.0, "c": 1.0}, + seed=7, + samples=1000, + ) + + assert result.estimate == pytest.approx(1.0) + assert result.confidence_level == 0.95 + with pytest.raises(ValueError, match="strictly positive"): + paired_bootstrap_ratio({"a": 1.0}, {"a": 0.0}, seed=7, samples=100) + + +def test_confirmatory_gate_uses_single_model_latency_baseline_and_reliability() -> None: + manifest, report = _confirmatory_report() + paid = report.paid_analysis + assert paid is not None + assert paid.latency_baseline_condition_id == "single_frontier" + + weak = report.model_copy( + update={ + "reliability": report.reliability.model_copy( + update={"raw_agreement": 0.79, "cohen_kappa": 0.59} + ) + } + ) + failed = evaluate_confirmatory_gates( + manifest=manifest, report=_rebind_evidence(weak) + ).failed_gates + assert "raw_agreement" in failed + assert "overall_kappa" in failed + + +def test_rotating_grader_pairs_still_produce_complete_reliability() -> None: + manifest, report = _confirmatory_report() + raw = [] + for index, judgment in enumerate(report.raw_judgments): + pair = index // 2 + raw.append( + judgment.model_copy( + update={ + "grader_id": f"grader-{pair % 3}-{index % 2}", + "judgment_id": f"rotated-{index}", + } + ) + ) + rescored = score_study( + manifest=manifest, + study_run=_confirmatory_report_run(manifest), + raw_judgments=tuple(raw), + bootstrap_seed=manifest.frozen_design.bootstrap.seed, + bootstrap_samples=manifest.frozen_design.bootstrap.samples, + ) + assert rescored.reliability.double_grading_rate == 1.0 + assert rescored.reliability.paired_judgments == len(manifest.planned_runs) From 5b03c18c40720cd9c6f4ad817f1a3144c89bf8a2 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 19:03:41 -0400 Subject: [PATCH 11/13] fix(evals): recompute confirmatory bounds from raw evidence --- src/conclave/evals/scoring.py | 138 ++++++++++++++---- tests/evals/test_analysis_gates.py | 223 ++++++++++++++++++++--------- 2 files changed, 265 insertions(+), 96 deletions(-) diff --git a/src/conclave/evals/scoring.py b/src/conclave/evals/scoring.py index a4f64ae..9b4293b 100644 --- a/src/conclave/evals/scoring.py +++ b/src/conclave/evals/scoring.py @@ -243,6 +243,18 @@ def validate_bonferroni_metadata(self) -> SimultaneousUpperBounds: return self +class SecondaryTaskStatistics(EvalModel): + """Immutable task-level sufficient inputs for all secondary bound calculations.""" + + task_id: str = Field(min_length=1) + elite_severe_error_rate: float = Field(ge=0, le=1) + baseline_severe_error_rate: float = Field(ge=0, le=1) + elite_readiness_error_rate: float = Field(ge=0, le=1) + baseline_readiness_error_rate: float = Field(ge=0, le=1) + elite_reviewer_effort: float = Field(gt=0) + baseline_reviewer_effort: float = Field(gt=0) + + class PaidAnalysisSummary(EvalModel): """Task-clustered operational and quality summaries for paid studies.""" @@ -260,6 +272,7 @@ class PaidAnalysisSummary(EvalModel): excluded_task_ids: tuple[str, ...] family_directions: dict[str, float] roster_directions: dict[str, float] + secondary_task_statistics: tuple[SecondaryTaskStatistics, ...] = Field(min_length=1) class ConfirmatoryGateResult(EvalModel): @@ -272,9 +285,13 @@ class ConfirmatoryGateResult(EvalModel): def evaluate_confirmatory_gates( *, manifest: StudyManifest, + study_run: StudyRun, + raw_judgments: Sequence[GraderJudgment], report: StudyScoreReport, + adjudications: Sequence[AdjudicationRecord] = (), + blind_map: BlindMap | None = None, ) -> ConfirmatoryGateResult: - """Derive every decision input from one frozen manifest and its bound score report.""" + """Re-score raw artifacts, reject report substitution, then evaluate frozen gates.""" design = manifest.frozen_design if manifest.evidence_classification != "confirmatory" or design is None: @@ -293,11 +310,27 @@ def evaluate_confirmatory_gates( raise ValueError("confirmatory report is missing paid analysis") if report.simultaneous_upper_bounds is None: raise ValueError("confirmatory report is missing simultaneous upper bounds") + derived_report = score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=raw_judgments, + adjudications=adjudications, + blind_map=blind_map, + bootstrap_seed=design.bootstrap.seed, + bootstrap_samples=design.bootstrap.samples, + ) + if report != derived_report: + raise ValueError("confirmatory report does not exactly match re-scored raw artifacts") + report = derived_report metrics = report.paid_analysis bounds = report.simultaneous_upper_bounds config = design.analysis_gates if bounds.evidence_hash != hash_confirmatory_evidence(report): raise ValueError("simultaneous-bound evidence hash does not match the report") + recomputed_bounds = recompute_simultaneous_upper_bounds(manifest=manifest, report=report) + if bounds != recomputed_bounds: + raise ValueError("simultaneous bounds do not match recomputed task-level inputs") + bounds = recomputed_bounds expected_per_endpoint_alpha = config.alpha / 3 if not math.isclose(bounds.per_endpoint_alpha, expected_per_endpoint_alpha, abs_tol=1e-15): raise ValueError("simultaneous-bound alpha does not match the frozen design") @@ -554,6 +587,65 @@ def hash_confirmatory_evidence(report: StudyScoreReport) -> str: return f"sha256:{hashlib.sha256(canonical).hexdigest()}" +def recompute_simultaneous_upper_bounds( + *, manifest: StudyManifest, report: StudyScoreReport +) -> SimultaneousUpperBounds: + """Recompute every secondary bound solely from archived task-level sufficient inputs.""" + + design = manifest.frozen_design + if design is None or report.paid_analysis is None: + raise ValueError("simultaneous bounds require a frozen paid analysis") + statistics = report.paid_analysis.secondary_task_statistics + by_task = {item.task_id: item for item in statistics} + if len(by_task) != len(statistics): + raise ValueError("secondary task statistics must contain unique task IDs") + expected_task_ids = {run.task_id for run in analysis_planned_runs(manifest)} + if set(by_task) != expected_task_ids: + raise ValueError("secondary task statistics must exactly cover the frozen analysis tasks") + + config = design.analysis_gates + per_endpoint_alpha = config.alpha / 3 + quantile_probability = 1 - per_endpoint_alpha + severe_elite = {task_id: item.elite_severe_error_rate for task_id, item in by_task.items()} + severe_baseline = { + task_id: item.baseline_severe_error_rate for task_id, item in by_task.items() + } + readiness_elite = { + task_id: item.elite_readiness_error_rate for task_id, item in by_task.items() + } + readiness_baseline = { + task_id: item.baseline_readiness_error_rate for task_id, item in by_task.items() + } + effort_elite = {task_id: item.elite_reviewer_effort for task_id, item in by_task.items()} + effort_baseline = {task_id: item.baseline_reviewer_effort for task_id, item in by_task.items()} + return SimultaneousUpperBounds( + per_endpoint_alpha=per_endpoint_alpha, + quantile_probability=quantile_probability, + severe_error_upper_bound=_bootstrap_difference_upper_bound( + severe_elite, + severe_baseline, + quantile_probability=quantile_probability, + seed=design.bootstrap.seed, + samples=design.bootstrap.samples, + ), + readiness_error_upper_bound=_bootstrap_difference_upper_bound( + readiness_elite, + readiness_baseline, + quantile_probability=quantile_probability, + seed=design.bootstrap.seed, + samples=design.bootstrap.samples, + ), + reviewer_effort_ratio_upper_bound=_bootstrap_median_ratio_upper_bound( + effort_elite, + effort_baseline, + quantile_probability=quantile_probability, + seed=design.bootstrap.seed, + samples=design.bootstrap.samples, + ), + evidence_hash=hash_confirmatory_evidence(report), + ) + + def cohen_kappa(first: Sequence[bool], second: Sequence[bool]) -> float | None: """Return Cohen's kappa for aligned binary ratings, or ``None`` if undefined.""" @@ -1009,6 +1101,18 @@ def selected(run, name): ) return values + secondary_task_statistics = tuple( + SecondaryTaskStatistics( + task_id=task_id, + elite_severe_error_rate=severe_means["elite_full"][task_id], + baseline_severe_error_rate=severe_means[baseline][task_id], + elite_readiness_error_rate=readiness_means["elite_full"][task_id], + baseline_readiness_error_rate=readiness_means[baseline][task_id], + elite_reviewer_effort=effort_means["elite_full"][task_id], + baseline_reviewer_effort=effort_means[baseline][task_id], + ) + for task_id in sorted(severe_means["elite_full"]) + ) paid_analysis = PaidAnalysisSummary( primary_paired_difference=primary, severe_error_difference=severe_difference, @@ -1026,6 +1130,7 @@ def selected(run, name): excluded_task_ids=tuple(sorted(excluded_task_ids)), family_directions=directional_effects("family"), roster_directions=directional_effects("roster"), + secondary_task_statistics=secondary_task_statistics, ) report = StudyScoreReport( study_id=manifest.study_id, @@ -1061,34 +1166,5 @@ def selected(run, name): if manifest.evidence_classification != "confirmatory": return report assert manifest.frozen_design is not None - design = manifest.frozen_design - config = design.analysis_gates - per_endpoint_alpha = config.alpha / 3 - quantile_probability = 1 - per_endpoint_alpha - bounds = SimultaneousUpperBounds( - per_endpoint_alpha=per_endpoint_alpha, - quantile_probability=quantile_probability, - severe_error_upper_bound=_bootstrap_difference_upper_bound( - severe_means["elite_full"], - severe_means[baseline], - quantile_probability=quantile_probability, - seed=design.bootstrap.seed, - samples=design.bootstrap.samples, - ), - readiness_error_upper_bound=_bootstrap_difference_upper_bound( - readiness_means["elite_full"], - readiness_means[baseline], - quantile_probability=quantile_probability, - seed=design.bootstrap.seed, - samples=design.bootstrap.samples, - ), - reviewer_effort_ratio_upper_bound=_bootstrap_median_ratio_upper_bound( - effort_means["elite_full"], - effort_means[baseline], - quantile_probability=quantile_probability, - seed=design.bootstrap.seed, - samples=design.bootstrap.samples, - ), - evidence_hash=hash_confirmatory_evidence(report), - ) + bounds = recompute_simultaneous_upper_bounds(manifest=manifest, report=report) return report.model_copy(update={"simultaneous_upper_bounds": bounds}) diff --git a/tests/evals/test_analysis_gates.py b/tests/evals/test_analysis_gates.py index 96069ab..be9708f 100644 --- a/tests/evals/test_analysis_gates.py +++ b/tests/evals/test_analysis_gates.py @@ -6,10 +6,12 @@ from conclave.evals.models import AnalysisGateConfig, RunRecord, StudyRun from conclave.evals.protocols import CONDITION_IDS, build_study_manifest from conclave.evals.scoring import ( + AtomicError, analysis_planned_runs, evaluate_confirmatory_gates, hash_confirmatory_evidence, paired_bootstrap_ratio, + recompute_simultaneous_upper_bounds, score_study, ) from tests.evals.test_method_grading import _judgment @@ -112,6 +114,15 @@ def _confirmatory_report_run(manifest): return study_run +def _gate(manifest, report): + return evaluate_confirmatory_gates( + manifest=manifest, + study_run=_confirmatory_report_run(manifest), + raw_judgments=report.raw_judgments, + report=report, + ) + + def _rebind_evidence(report): bounds = report.simultaneous_upper_bounds assert bounds is not None @@ -126,16 +137,11 @@ def _rebind_evidence(report): def test_confirmatory_gate_is_bound_to_manifest_report_and_simultaneous_bounds() -> None: manifest, report = _confirmatory_report() - result = evaluate_confirmatory_gates(manifest=manifest, report=report) + result = _gate(manifest, report) assert result.status == "GO" undersized_manifest, undersized_report = _confirmatory_report(minimum_tasks=3) - assert ( - "sample_size" - in evaluate_confirmatory_gates( - manifest=undersized_manifest, report=undersized_report - ).failed_gates - ) + assert "sample_size" in _gate(undersized_manifest, undersized_report).failed_gates assert report.simultaneous_upper_bounds is not None assert report.simultaneous_upper_bounds.method == "task_clustered_percentile_bonferroni_v1" @@ -149,8 +155,8 @@ def test_confirmatory_gate_is_bound_to_manifest_report_and_simultaneous_bounds() ) } ) - with pytest.raises(ValueError, match="evidence hash"): - evaluate_confirmatory_gates(manifest=manifest, report=tampered) + with pytest.raises(ValueError, match="re-scored raw artifacts"): + _gate(manifest, tampered) def test_confirmatory_gate_uses_every_derived_report_boundary() -> None: @@ -172,23 +178,6 @@ def test_confirmatory_gate_uses_every_derived_report_boundary() -> None: ) } ), - "severe_error_noninferiority": paid.model_copy( - update={"severe_error_difference": paid.severe_error_difference} - ), - "readiness_noninferiority": paid.model_copy( - update={ - "readiness_error_difference": paid.readiness_error_difference.model_copy( - update={"upper": 0.051} - ) - } - ), - "reviewer_effort": paid.model_copy( - update={ - "reviewer_effort_ratio_interval": paid.reviewer_effort_ratio_interval.model_copy( - update={"upper": 1.201} - ) - } - ), "latency_ratio": paid.model_copy(update={"p95_latency_ratio": 3.001}), "latency_ceiling": paid.model_copy(update={"elite_p95_latency_seconds": 180.1}), "spend_ceiling": paid.model_copy(update={"total_cost_usd": 251.0}), @@ -199,59 +188,166 @@ def test_confirmatory_gate_uses_every_derived_report_boundary() -> None: update={"roster_directions": {"roster-a": 0.0, "roster-b": 1.0}} ), } - for expected_gate, changed in cases.items(): + for changed in cases.values(): changed_report = report.model_copy(update={"paid_analysis": changed}) - if expected_gate in { + changed_report = _rebind_evidence(changed_report) + with pytest.raises(ValueError, match="re-scored raw artifacts"): + _gate(manifest, changed_report) + + +@pytest.mark.parametrize( + ("expected_gate", "updates"), + ( + ( "severe_error_noninferiority", + {"elite_severe_error_rate": 1.0, "baseline_severe_error_rate": 0.0}, + ), + ( "readiness_noninferiority", + {"elite_readiness_error_rate": 1.0, "baseline_readiness_error_rate": 0.0}, + ), + ( "reviewer_effort", - }: - bounds = changed_report.simultaneous_upper_bounds - assert bounds is not None - field = { - "severe_error_noninferiority": "severe_error_upper_bound", - "readiness_noninferiority": "readiness_error_upper_bound", - "reviewer_effort": "reviewer_effort_ratio_upper_bound", - }[expected_gate] - changed_report = changed_report.model_copy( - update={"simultaneous_upper_bounds": bounds.model_copy(update={field: 2.0})} + {"elite_reviewer_effort": 25.0, "baseline_reviewer_effort": 10.0}, + ), + ), +) +def test_confirmatory_gate_recomputes_secondary_bounds_from_task_inputs( + expected_gate, updates +) -> None: + manifest, report = _confirmatory_report() + paid = report.paid_analysis + assert paid is not None + changed_paid = paid.model_copy( + update={ + "secondary_task_statistics": tuple( + item.model_copy(update=updates) for item in paid.secondary_task_statistics ) - changed_report = _rebind_evidence(changed_report) - result = evaluate_confirmatory_gates( + } + ) + changed_report = report.model_copy(update={"paid_analysis": changed_paid}) + changed_report = changed_report.model_copy( + update={ + "simultaneous_upper_bounds": recompute_simultaneous_upper_bounds( + manifest=manifest, report=changed_report + ) + } + ) + + bounds = changed_report.simultaneous_upper_bounds + assert bounds is not None + values = { + "severe_error_noninferiority": bounds.severe_error_upper_bound, + "readiness_noninferiority": bounds.readiness_error_upper_bound, + "reviewer_effort": bounds.reviewer_effort_ratio_upper_bound, + } + thresholds = { + "severe_error_noninferiority": manifest.frozen_design.analysis_gates.severe_error_noninferiority_margin, + "readiness_noninferiority": manifest.frozen_design.analysis_gates.readiness_noninferiority_margin, + "reviewer_effort": manifest.frozen_design.analysis_gates.reviewer_effort_max_ratio, + } + assert values[expected_gate] > thresholds[expected_gate] + with pytest.raises(ValueError, match="re-scored raw artifacts"): + _gate(manifest, changed_report) + + +def test_caller_cannot_substitute_optimistic_bounds_to_turn_redesign_into_go() -> None: + manifest, report = _confirmatory_report() + condition_by_run = { + planned.planned_run_id: planned.condition_id for planned in manifest.planned_runs + } + severe = AtomicError( + rubric_item="safety", + category="severe_harm", + severity="severe", + detail="Confirmatory severe error.", + ) + bad_judgments = tuple( + judgment.model_copy( + update={ + "critical_error_free": False, + "atomic_errors": (severe,), + "severe_error": True, + } + ) + if condition_by_run[judgment.planned_run_id] == "elite_full" + else judgment + for judgment in report.raw_judgments + ) + study_run = _confirmatory_report_run(manifest) + redesign_report = score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=bad_judgments, + bootstrap_seed=manifest.frozen_design.bootstrap.seed, + bootstrap_samples=manifest.frozen_design.bootstrap.samples, + ) + assert ( + evaluate_confirmatory_gates( manifest=manifest, - report=changed_report, + study_run=study_run, + raw_judgments=bad_judgments, + report=redesign_report, + ).status + == "REDESIGN" + ) + + bounds = redesign_report.simultaneous_upper_bounds + paid = redesign_report.paid_analysis + assert bounds is not None and paid is not None + substituted = redesign_report.model_copy( + update={ + "paid_analysis": paid.model_copy( + update={ + "secondary_task_statistics": tuple( + item.model_copy( + update={ + "elite_severe_error_rate": 0.0, + "baseline_severe_error_rate": 0.0, + } + ) + for item in paid.secondary_task_statistics + ) + } + ), + "simultaneous_upper_bounds": bounds.model_copy( + update={"severe_error_upper_bound": 0.0} + ), + } + ) + substituted = _rebind_evidence(substituted) + with pytest.raises(ValueError, match="re-scored raw artifacts"): + evaluate_confirmatory_gates( + manifest=manifest, + study_run=study_run, + raw_judgments=bad_judgments, + report=substituted, ) - assert expected_gate in result.failed_gates def test_confirmatory_gate_refuses_exploratory_missing_bounds_or_provenance_drift() -> None: manifest, report = _confirmatory_report() with pytest.raises(ValueError, match="confirmatory"): - evaluate_confirmatory_gates( - manifest=manifest.model_copy( - update={"evidence_classification": "paid_exploratory_pilot"} - ), - report=report, + _gate( + manifest.model_copy(update={"evidence_classification": "paid_exploratory_pilot"}), + report, ) with pytest.raises(ValueError, match="simultaneous"): - evaluate_confirmatory_gates( - manifest=manifest, - report=report.model_copy(update={"simultaneous_upper_bounds": None}), - ) + _gate(manifest, report.model_copy(update={"simultaneous_upper_bounds": None})) with pytest.raises(ValueError, match="frozen design hash"): - evaluate_confirmatory_gates( - manifest=manifest, - report=report.model_copy(update={"frozen_design_hash": "sha256:" + "c" * 64}), + _gate( + manifest, + report.model_copy(update={"frozen_design_hash": "sha256:" + "c" * 64}), ) with pytest.raises(ValueError, match="preregistration hash"): - evaluate_confirmatory_gates( - manifest=manifest, - report=report.model_copy(update={"preregistration_hash": "sha256:" + "c" * 64}), + _gate( + manifest, + report.model_copy(update={"preregistration_hash": "sha256:" + "c" * 64}), ) with pytest.raises(ValueError, match="analysis code hash"): - evaluate_confirmatory_gates( - manifest=manifest, - report=report.model_copy(update={"analysis_code_hash": "sha256:" + "c" * 64}), + _gate( + manifest, + report.model_copy(update={"analysis_code_hash": "sha256:" + "c" * 64}), ) @@ -303,11 +399,8 @@ def test_confirmatory_gate_uses_single_model_latency_baseline_and_reliability() ) } ) - failed = evaluate_confirmatory_gates( - manifest=manifest, report=_rebind_evidence(weak) - ).failed_gates - assert "raw_agreement" in failed - assert "overall_kappa" in failed + with pytest.raises(ValueError, match="re-scored raw artifacts"): + _gate(manifest, _rebind_evidence(weak)) def test_rotating_grader_pairs_still_produce_complete_reliability() -> None: From 58552990dd7f4697020668e4d4c5bf1d4c9fae10 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Fri, 17 Jul 2026 18:58:39 -0400 Subject: [PATCH 12/13] fix(evals): make rotating grader kappa invariant --- src/conclave/evals/scoring.py | 49 ++++++++++++++++++++++++------ tests/evals/test_method_grading.py | 39 ++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/src/conclave/evals/scoring.py b/src/conclave/evals/scoring.py index 9b4293b..97e76ad 100644 --- a/src/conclave/evals/scoring.py +++ b/src/conclave/evals/scoring.py @@ -153,6 +153,7 @@ class ReliabilityMetrics(EvalModel): grader_pair: tuple[str, str] | None = None cohen_kappa: float | None = Field(default=None, ge=-1.0, le=1.0) + kappa_method: Literal["cohen_fixed_pair", "fleiss_rotating_pairs"] | None = None paired_judgments: int = Field(ge=0) disagreements: int = Field(ge=0) adjudicated_disagreements: int = Field(ge=0) @@ -174,6 +175,7 @@ class ReliabilityStratum(EvalModel): raw_agreement: float | None = Field(default=None, ge=0.0, le=1.0) positive_prevalence: float | None = Field(default=None, ge=0.0, le=1.0) cohen_kappa: float | None = Field(default=None, ge=-1.0, le=1.0) + kappa_method: Literal["cohen_fixed_pair", "fleiss_rotating_pairs"] | None = None class StudyScoreReport(EvalModel): @@ -660,6 +662,19 @@ def cohen_kappa(first: Sequence[bool], second: Sequence[bool]) -> float | None: return (observed - expected) / (1 - expected) +def fleiss_kappa(pairs: Sequence[tuple[bool, bool]]) -> float | None: + """Return label-invariant Fleiss kappa for two ratings per item.""" + + if not pairs: + return None + observed = sum(left == right for left, right in pairs) / len(pairs) + positive_prevalence = sum(left + right for left, right in pairs) / (2 * len(pairs)) + expected = positive_prevalence**2 + (1 - positive_prevalence) ** 2 + if expected == 1.0: + return None + return (observed - expected) / (1 - expected) + + def _target_map(blind_map: BlindMap | None) -> dict[str, str]: if blind_map is None: return {} @@ -682,7 +697,8 @@ def _planned_target(record: GraderJudgment | AdjudicationRecord, mapping: Mappin def _paired_values( judgments_by_run: Mapping[str, Sequence[GraderJudgment]], run_ids: Sequence[str] ) -> tuple[tuple[str, str] | None, list[tuple[bool, bool]]]: - graders = sorted({item.grader_id for values in judgments_by_run.values() for item in values}) + relevant = [item for run_id in run_ids for item in judgments_by_run.get(run_id, ())] + graders = sorted({item.grader_id for item in relevant}) grader_pair = (graders[0], graders[1]) if len(graders) == 2 else None pairs = [] for run_id in sorted(run_ids): @@ -694,14 +710,22 @@ def _paired_values( def _agreement_statistics( pairs: Sequence[tuple[bool, bool]], -) -> tuple[float | None, float | None, float | None]: + grader_pair: tuple[str, str] | None, +) -> tuple[ + float | None, + float | None, + float | None, + Literal["cohen_fixed_pair", "fleiss_rotating_pairs"] | None, +]: if not pairs: - return None, None, None + return None, None, None, None first = [left for left, _ in pairs] second = [right for _, right in pairs] raw_agreement = sum(left == right for left, right in pairs) / len(pairs) prevalence = (sum(first) + sum(second)) / (2 * len(pairs)) - return raw_agreement, prevalence, cohen_kappa(first, second) + if grader_pair is not None: + return raw_agreement, prevalence, cohen_kappa(first, second), "cohen_fixed_pair" + return raw_agreement, prevalence, fleiss_kappa(pairs), "fleiss_rotating_pairs" def _reliability( @@ -727,7 +751,7 @@ def _reliability( ) rate = adjudicated_disagreements / disagreements if disagreements else 0.0 grader_pair, pairs = _paired_values(judgments_by_run, tuple(judgments_by_run)) - raw_agreement, prevalence, kappa = _agreement_statistics(pairs) + raw_agreement, prevalence, kappa, kappa_method = _agreement_statistics(pairs, grader_pair) strata: list[ReliabilityStratum] = [] if manifest.frozen_design is not None: for family in sorted(set(manifest.frozen_design.task_family_map.values())): @@ -736,8 +760,10 @@ def _reliability( for run in manifest.planned_runs if manifest.frozen_design.task_family_map[run.task_id] == family ] - _, stratum_pairs = _paired_values(judgments_by_run, run_ids) - agreement, stratum_prevalence, stratum_kappa = _agreement_statistics(stratum_pairs) + stratum_pair, stratum_pairs = _paired_values(judgments_by_run, run_ids) + agreement, stratum_prevalence, stratum_kappa, stratum_method = _agreement_statistics( + stratum_pairs, stratum_pair + ) strata.append( ReliabilityStratum( stratum_type="family", @@ -746,14 +772,17 @@ def _reliability( raw_agreement=agreement, positive_prevalence=stratum_prevalence, cohen_kappa=stratum_kappa, + kappa_method=stratum_method, ) ) for roster_id in sorted(roster.roster_id for roster in manifest.frozen_design.rosters): run_ids = [ run.planned_run_id for run in manifest.planned_runs if run.roster_id == roster_id ] - _, stratum_pairs = _paired_values(judgments_by_run, run_ids) - agreement, stratum_prevalence, stratum_kappa = _agreement_statistics(stratum_pairs) + stratum_pair, stratum_pairs = _paired_values(judgments_by_run, run_ids) + agreement, stratum_prevalence, stratum_kappa, stratum_method = _agreement_statistics( + stratum_pairs, stratum_pair + ) strata.append( ReliabilityStratum( stratum_type="roster", @@ -762,11 +791,13 @@ def _reliability( raw_agreement=agreement, positive_prevalence=stratum_prevalence, cohen_kappa=stratum_kappa, + kappa_method=stratum_method, ) ) return ReliabilityMetrics( grader_pair=grader_pair, cohen_kappa=kappa, + kappa_method=kappa_method, paired_judgments=len(pairs), disagreements=disagreements, adjudicated_disagreements=adjudicated_disagreements, diff --git a/tests/evals/test_method_grading.py b/tests/evals/test_method_grading.py index 0b2db21..3322355 100644 --- a/tests/evals/test_method_grading.py +++ b/tests/evals/test_method_grading.py @@ -319,6 +319,7 @@ def test_paid_reliability_reports_raw_agreement_prevalence_and_strata() -> None: ) assert report.evidence_classification == "paid_exploratory_pilot" + assert report.reliability.kappa_method == "cohen_fixed_pair" assert report.reliability.raw_agreement == pytest.approx( (len(successful) - 1) / len(successful) ) @@ -336,3 +337,41 @@ def test_paid_reliability_reports_raw_agreement_prevalence_and_strata() -> None: def test_constant_prevalence_kappa_is_undefined_not_perfect() -> None: assert cohen_kappa([True, True], [True, True]) is None + + +def test_rotating_grader_kappa_is_label_invariant_and_stratified() -> None: + manifest, study_run = _paid_study() + successful = [record for record in study_run.records if record.outcome == "success"] + pairs = (("A", "B"), ("B", "C"), ("A", "C"), ("A", "B")) + ratings = ((False, False), (False, True), (False, False), (False, True)) + + def judgments(rename_b: bool): + values = [] + for index, record in enumerate(successful): + pair = pairs[index % len(pairs)] + outcomes = ratings[index % len(ratings)] + for grader, outcome in zip(pair, outcomes, strict=True): + grader_id = "Z" if rename_b and grader == "B" else grader + values.append(_judgment(record.planned_run_id, grader_id, value=outcome)) + return tuple(values) + + original = score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments(False), + bootstrap_seed=19, + bootstrap_samples=100, + ) + renamed = score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments(True), + bootstrap_seed=19, + bootstrap_samples=100, + ) + + assert original.reliability.grader_pair is None + assert original.reliability.kappa_method == "fleiss_rotating_pairs" + assert original.reliability.cohen_kappa == pytest.approx(renamed.reliability.cohen_kappa) + assert original.reliability.raw_agreement == renamed.reliability.raw_agreement + assert all(item.kappa_method is not None for item in original.reliability.strata) From 1b67e81fb16cbc44701fbc12aa76105af1ea39e4 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Sat, 18 Jul 2026 20:06:45 -0400 Subject: [PATCH 13/13] fix(evals): enforce paid blinding boundary --- .../2026-07-17-h1-method-hardening-design.md | 10 +- docs/plans/2026-07-17-h1-method-hardening.md | 2 +- src/conclave/evals/scoring.py | 12 ++ tests/evals/test_analysis_gates.py | 33 +++-- tests/evals/test_method_grading.py | 121 ++++++++++++++++-- 5 files changed, 153 insertions(+), 25 deletions(-) diff --git a/docs/plans/2026-07-17-h1-method-hardening-design.md b/docs/plans/2026-07-17-h1-method-hardening-design.md index e4e5012..539435c 100644 --- a/docs/plans/2026-07-17-h1-method-hardening-design.md +++ b/docs/plans/2026-07-17-h1-method-hardening-design.md @@ -1,7 +1,7 @@ # H1 Method Hardening Design -**Linear:** DSE-708 -**Dependency:** PR #52 at `4365c28` +**Linear:** DSE-708 +**Dependency:** PR #52 at `4365c28` **Gate:** Required before any paid study call ## Decision @@ -19,17 +19,17 @@ Confirmatory reports fail closed if any frozen field is missing or its observed ## Experimental unit and randomization -The unit of analysis is the task. Roster and replicate are repeated measurements averaged within task. Create every task x roster x condition x replicate cell before execution. Derive an independent condition permutation from the master seed plus task and roster IDs; interleave blocks so time or provider order cannot align with one condition. All failed, timed-out, malformed, over-budget, abstained, incomplete, ungraded, and unresolved cells remain failures. +The unit of analysis is the task. Roster and replicate are repeated measurements averaged within task. Create every task x roster x condition x replicate cell before execution. Derive an independent condition permutation from the master seed plus task and roster IDs so condition order varies across blocks. All failed, timed-out, malformed, over-budget, abstained, incomplete, ungraded, and unresolved cells remain failures. ## Grading contract Two independent graders score every successful output. Atomic records include rubric item, error category and severity, severe-error flag, holistic dimensions, reviewer seconds, confidence or abstention, rubric version/hash, grader batch/order, and condition/provider guess. Adjudication sees only genuine disagreements, cites every source record, and never overwrites raw judgments. -Non-success records bypass the human queue and are automatic failures. Successful grader views strip provider, model, condition, mode, and formatting fingerprints; normalize presentation deterministically; run a leakage scan; and retain a separate hashed/access-controlled blind map. +Non-success records bypass the human queue and are automatic failures. Successful grader views expose only normalized presentation text, scan it for predeclared provider/model/condition labels, and retain a separate hashed/access-controlled blind map. Paid and confirmatory scoring rejects direct planned-run targets, unhashed maps, and maps that do not cover exactly the successful output set. ## Analysis and gates -Study reporting uses failure-inclusive Wilson rates, task-clustered paired bootstrap differences with roster/replicate averaging, raw agreement, prevalence, kappa, adjudication rate, severe errors, reviewer effort, token/cost distributions, latency distributions, deviations, and leakage guesses. +Study reporting uses failure-inclusive Wilson rates, task-clustered paired bootstrap differences with roster/replicate averaging, raw agreement, prevalence, kappa, adjudication rate, severe errors, reviewer effort, total and analysis-set cost, p95 latency, deviations, and retained leakage guesses. A paid workflow must preregister and apply its material-leakage rule before claiming method readiness. Reliability is undefined—not perfect—when expected agreement is one. A future paid pilot is method-ready only with at least 95% double grading, raw agreement at least 80%, kappa at least diff --git a/docs/plans/2026-07-17-h1-method-hardening.md b/docs/plans/2026-07-17-h1-method-hardening.md index 8fe63af..5c19afc 100644 --- a/docs/plans/2026-07-17-h1-method-hardening.md +++ b/docs/plans/2026-07-17-h1-method-hardening.md @@ -16,7 +16,7 @@ ## Phase 3 — Confirmatory estimands and refusal gates -1. Add tests for task-level roster averaging, task-clustered paired bootstrap, one-sided severe-error and effort/latency noninferiority boundaries, symmetric task exclusions, deviations, cost/latency distributions, and report refusal on freeze drift or exploratory evidence. +1. Add tests for task-level roster averaging, task-clustered paired bootstrap, one-sided severe-error and effort/latency noninferiority boundaries, symmetric task exclusions, deviations, complete cost/latency receipts and summaries, and report refusal on freeze drift or exploratory evidence. 2. Extend `scoring.py`, `reporting.py`, and CLI artifact validation. Archive every gate input and seed. 3. Re-run focused tests and commit. diff --git a/src/conclave/evals/scoring.py b/src/conclave/evals/scoring.py index 97e76ad..ccbd405 100644 --- a/src/conclave/evals/scoring.py +++ b/src/conclave/evals/scoring.py @@ -912,7 +912,19 @@ def score_study( if len({item.adjudication_id for item in adjudications}) != len(adjudications): raise ValueError("adjudication_id values must be unique") + paid_study = manifest.evidence_classification != "synthetic_exploratory" + if paid_study: + if blind_map is None or blind_map.blind_map_hash is None: + raise ValueError("paid scoring requires a hashed blind map") + if any(item.planned_run_id is not None for item in (*raw_judgments, *adjudications)): + raise ValueError("paid judgments and adjudications require opaque output IDs") mapping = _target_map(blind_map) + if paid_study: + successful_run_ids = { + record.planned_run_id for record in study_run.records if record.outcome == "success" + } + if set(mapping.values()) != successful_run_ids or len(mapping) != len(successful_run_ids): + raise ValueError("paid blind map must cover exactly the successful output set") judgments_by_run: dict[str, list[GraderJudgment]] = defaultdict(list) judgment_ids = {item.judgment_id for item in raw_judgments} seen_grader_targets: set[tuple[str, str]] = set() diff --git a/tests/evals/test_analysis_gates.py b/tests/evals/test_analysis_gates.py index be9708f..dbc9410 100644 --- a/tests/evals/test_analysis_gates.py +++ b/tests/evals/test_analysis_gates.py @@ -14,7 +14,7 @@ recompute_simultaneous_upper_bounds, score_study, ) -from tests.evals.test_method_grading import _judgment +from tests.evals.test_method_grading import _blind_targets, _judgment from tests.evals.test_study_design import _design, _tasks @@ -74,9 +74,14 @@ def _confirmatory_report(*, minimum_tasks: int = 2): frozen_design=design, ) study_run = _confirmatory_report_run(manifest) + blind_map, opaque_by_run = _blind_targets(study_run) records = study_run.records judgments = tuple( - _judgment(record.planned_run_id, grader, value=planned.condition_id != "self_refine") + _judgment( + opaque_by_run[record.planned_run_id], + grader, + value=planned.condition_id != "self_refine", + ) for planned, record in zip(manifest.planned_runs, records, strict=True) for grader in ("grader-a", "grader-b") ) @@ -84,6 +89,7 @@ def _confirmatory_report(*, minimum_tasks: int = 2): manifest=manifest, study_run=study_run, raw_judgments=judgments, + blind_map=blind_map, bootstrap_seed=design.bootstrap.seed, bootstrap_samples=design.bootstrap.samples, ) @@ -115,11 +121,14 @@ def _confirmatory_report_run(manifest): def _gate(manifest, report): + study_run = _confirmatory_report_run(manifest) + blind_map, _ = _blind_targets(study_run) return evaluate_confirmatory_gates( manifest=manifest, - study_run=_confirmatory_report_run(manifest), + study_run=study_run, raw_judgments=report.raw_judgments, report=report, + blind_map=blind_map, ) @@ -253,8 +262,11 @@ def test_confirmatory_gate_recomputes_secondary_bounds_from_task_inputs( def test_caller_cannot_substitute_optimistic_bounds_to_turn_redesign_into_go() -> None: manifest, report = _confirmatory_report() - condition_by_run = { - planned.planned_run_id: planned.condition_id for planned in manifest.planned_runs + study_run = _confirmatory_report_run(manifest) + blind_map, opaque_by_run = _blind_targets(study_run) + condition_by_output = { + opaque_by_run[planned.planned_run_id]: planned.condition_id + for planned in manifest.planned_runs } severe = AtomicError( rubric_item="safety", @@ -270,15 +282,15 @@ def test_caller_cannot_substitute_optimistic_bounds_to_turn_redesign_into_go() - "severe_error": True, } ) - if condition_by_run[judgment.planned_run_id] == "elite_full" + if condition_by_output[judgment.opaque_output_id] == "elite_full" else judgment for judgment in report.raw_judgments ) - study_run = _confirmatory_report_run(manifest) redesign_report = score_study( manifest=manifest, study_run=study_run, raw_judgments=bad_judgments, + blind_map=blind_map, bootstrap_seed=manifest.frozen_design.bootstrap.seed, bootstrap_samples=manifest.frozen_design.bootstrap.samples, ) @@ -288,6 +300,7 @@ def test_caller_cannot_substitute_optimistic_bounds_to_turn_redesign_into_go() - study_run=study_run, raw_judgments=bad_judgments, report=redesign_report, + blind_map=blind_map, ).status == "REDESIGN" ) @@ -322,6 +335,7 @@ def test_caller_cannot_substitute_optimistic_bounds_to_turn_redesign_into_go() - study_run=study_run, raw_judgments=bad_judgments, report=substituted, + blind_map=blind_map, ) @@ -405,6 +419,8 @@ def test_confirmatory_gate_uses_single_model_latency_baseline_and_reliability() def test_rotating_grader_pairs_still_produce_complete_reliability() -> None: manifest, report = _confirmatory_report() + study_run = _confirmatory_report_run(manifest) + blind_map, _ = _blind_targets(study_run) raw = [] for index, judgment in enumerate(report.raw_judgments): pair = index // 2 @@ -418,8 +434,9 @@ def test_rotating_grader_pairs_still_produce_complete_reliability() -> None: ) rescored = score_study( manifest=manifest, - study_run=_confirmatory_report_run(manifest), + study_run=study_run, raw_judgments=tuple(raw), + blind_map=blind_map, bootstrap_seed=manifest.frozen_design.bootstrap.seed, bootstrap_samples=manifest.frozen_design.bootstrap.samples, ) diff --git a/tests/evals/test_method_grading.py b/tests/evals/test_method_grading.py index 3322355..c93d445 100644 --- a/tests/evals/test_method_grading.py +++ b/tests/evals/test_method_grading.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError +from conclave.evals.blinding import BlindMap, BlindMapEntry, build_grader_queue, hash_blind_map from conclave.evals.models import ( AnalysisGateConfig, BootstrapConfig, @@ -107,10 +108,15 @@ def _paid_study(): return manifest, study_run -def _judgment(run_id: str, grader: str, *, value: bool = True) -> GraderJudgment: +def _judgment(target_id: str, grader: str, *, value: bool = True) -> GraderJudgment: + target = ( + {"planned_run_id": target_id} + if target_id.startswith("run_") + else {"opaque_output_id": target_id} + ) return GraderJudgment( - judgment_id=f"{run_id}-{grader}", - planned_run_id=run_id, + judgment_id=f"{target_id}-{grader}", + **target, grader_id=grader, critical_error_free=value, atomic_errors=(), @@ -136,6 +142,11 @@ def _judgment(run_id: str, grader: str, *, value: bool = True) -> GraderJudgment ) +def _blind_targets(study_run: StudyRun) -> tuple[BlindMap, dict[str, str]]: + _, blind_map = build_grader_queue(study_run.records, seed=31, forbidden_labels=()) + return blind_map, {entry.planned_run_id: entry.opaque_output_id for entry in blind_map.entries} + + @pytest.mark.parametrize( "dimension", ( @@ -198,8 +209,9 @@ def test_atomic_errors_and_judgment_status_are_typed_and_consistent() -> None: def test_paid_scoring_requires_two_complete_independent_judgments_per_success() -> None: manifest, study_run = _paid_study() successful = [record for record in study_run.records if record.outcome == "success"] + blind_map, opaque_by_run = _blind_targets(study_run) complete = tuple( - _judgment(record.planned_run_id, grader) + _judgment(opaque_by_run[record.planned_run_id], grader) for record in successful for grader in ("grader-a", "grader-b") ) @@ -209,6 +221,7 @@ def test_paid_scoring_requires_two_complete_independent_judgments_per_success() manifest=manifest, study_run=study_run, raw_judgments=complete[:-1], + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, ) @@ -218,20 +231,41 @@ def test_paid_scoring_requires_two_complete_independent_judgments_per_success() manifest=manifest, study_run=study_run, raw_judgments=(incomplete, *complete[1:]), + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, ) - with pytest.raises(ValueError, match="non-success"): + with pytest.raises(ValueError, match="missing from the blind map"): score_study( manifest=manifest, study_run=study_run, - raw_judgments=(*complete, _judgment(study_run.records[0].planned_run_id, "grader-a")), + raw_judgments=(*complete, _judgment("output_" + "f" * 24, "grader-a")), + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, ) -def test_paid_scoring_requires_complete_cost_and_latency_receipts() -> None: +def test_paid_scoring_requires_a_hashed_blind_map() -> None: + manifest, study_run = _paid_study() + successful = [record for record in study_run.records if record.outcome == "success"] + judgments = tuple( + _judgment(record.planned_run_id, grader) + for record in successful + for grader in ("grader-a", "grader-b") + ) + + with pytest.raises(ValueError, match="hashed blind map"): + score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments, + bootstrap_seed=19, + bootstrap_samples=100, + ) + + +def test_paid_scoring_requires_opaque_judgment_targets() -> None: manifest, study_run = _paid_study() successful = [record for record in study_run.records if record.outcome == "success"] judgments = tuple( @@ -239,6 +273,58 @@ def test_paid_scoring_requires_complete_cost_and_latency_receipts() -> None: for record in successful for grader in ("grader-a", "grader-b") ) + _, blind_map = build_grader_queue(study_run.records, seed=31, forbidden_labels=()) + + with pytest.raises(ValueError, match="opaque output IDs"): + score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments, + blind_map=blind_map, + bootstrap_seed=19, + bootstrap_samples=100, + ) + + +def test_paid_scoring_rejects_blind_maps_outside_the_successful_output_set() -> None: + manifest, study_run = _paid_study() + _, blind_map = build_grader_queue(study_run.records, seed=31, forbidden_labels=()) + opaque_by_run = {entry.planned_run_id: entry.opaque_output_id for entry in blind_map.entries} + judgments = tuple( + _judgment(opaque_by_run[record.planned_run_id], grader) + for record in study_run.records + if record.outcome == "success" + for grader in ("grader-a", "grader-b") + ) + entries = ( + *blind_map.entries, + BlindMapEntry( + opaque_output_id="output_" + "f" * 24, + planned_run_id=study_run.records[0].planned_run_id, + ), + ) + expanded_map = BlindMap(entries=entries, blind_map_hash=hash_blind_map(entries)) + + with pytest.raises(ValueError, match="successful output set"): + score_study( + manifest=manifest, + study_run=study_run, + raw_judgments=judgments, + blind_map=expanded_map, + bootstrap_seed=19, + bootstrap_samples=100, + ) + + +def test_paid_scoring_requires_complete_cost_and_latency_receipts() -> None: + manifest, study_run = _paid_study() + successful = [record for record in study_run.records if record.outcome == "success"] + blind_map, opaque_by_run = _blind_targets(study_run) + judgments = tuple( + _judgment(opaque_by_run[record.planned_run_id], grader) + for record in successful + for grader in ("grader-a", "grader-b") + ) missing_latency = study_run.model_copy( update={ "records": ( @@ -252,6 +338,7 @@ def test_paid_scoring_requires_complete_cost_and_latency_receipts() -> None: manifest=manifest, study_run=missing_latency, raw_judgments=judgments, + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, ) @@ -268,6 +355,7 @@ def test_paid_scoring_requires_complete_cost_and_latency_receipts() -> None: manifest=manifest, study_run=missing_cost, raw_judgments=judgments, + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, ) @@ -276,8 +364,9 @@ def test_paid_scoring_requires_complete_cost_and_latency_receipts() -> None: def test_paid_scoring_uses_frozen_bootstrap_and_reports_effort_interval_and_cost_sets() -> None: manifest, study_run = _paid_study() successful = [record for record in study_run.records if record.outcome == "success"] + blind_map, opaque_by_run = _blind_targets(study_run) judgments = tuple( - _judgment(record.planned_run_id, grader) + _judgment(opaque_by_run[record.planned_run_id], grader) for record in successful for grader in ("grader-a", "grader-b") ) @@ -286,6 +375,7 @@ def test_paid_scoring_uses_frozen_bootstrap_and_reports_effort_interval_and_cost manifest=manifest, study_run=study_run, raw_judgments=judgments, + blind_map=blind_map, bootstrap_seed=7, bootstrap_samples=20, ) @@ -293,6 +383,7 @@ def test_paid_scoring_uses_frozen_bootstrap_and_reports_effort_interval_and_cost manifest=manifest, study_run=study_run, raw_judgments=judgments, + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, ) @@ -305,15 +396,18 @@ def test_paid_scoring_uses_frozen_bootstrap_and_reports_effort_interval_and_cost def test_paid_reliability_reports_raw_agreement_prevalence_and_strata() -> None: manifest, study_run = _paid_study() successful = [record for record in study_run.records if record.outcome == "success"] + blind_map, opaque_by_run = _blind_targets(study_run) judgments = [] for index, record in enumerate(successful): - judgments.append(_judgment(record.planned_run_id, "grader-a")) - judgments.append(_judgment(record.planned_run_id, "grader-b", value=index != 0)) + target_id = opaque_by_run[record.planned_run_id] + judgments.append(_judgment(target_id, "grader-a")) + judgments.append(_judgment(target_id, "grader-b", value=index != 0)) report = score_study( manifest=manifest, study_run=study_run, raw_judgments=tuple(judgments), + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, ) @@ -342,6 +436,7 @@ def test_constant_prevalence_kappa_is_undefined_not_perfect() -> None: def test_rotating_grader_kappa_is_label_invariant_and_stratified() -> None: manifest, study_run = _paid_study() successful = [record for record in study_run.records if record.outcome == "success"] + blind_map, opaque_by_run = _blind_targets(study_run) pairs = (("A", "B"), ("B", "C"), ("A", "C"), ("A", "B")) ratings = ((False, False), (False, True), (False, False), (False, True)) @@ -352,13 +447,16 @@ def judgments(rename_b: bool): outcomes = ratings[index % len(ratings)] for grader, outcome in zip(pair, outcomes, strict=True): grader_id = "Z" if rename_b and grader == "B" else grader - values.append(_judgment(record.planned_run_id, grader_id, value=outcome)) + values.append( + _judgment(opaque_by_run[record.planned_run_id], grader_id, value=outcome) + ) return tuple(values) original = score_study( manifest=manifest, study_run=study_run, raw_judgments=judgments(False), + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, ) @@ -366,6 +464,7 @@ def judgments(rename_b: bool): manifest=manifest, study_run=study_run, raw_judgments=judgments(True), + blind_map=blind_map, bootstrap_seed=19, bootstrap_samples=100, )