Fix OpenAI strict verdict extraction schema

docs/PRODUCT_DESIGN_DOCUMENT.md

@@ -227,8 +227,9 @@ repair retry. Capable providers **enforce the schema at decode time** — OpenAI
 `response_format` json_schema, Gemini `responseSchema`, Anthropic tool `input_schema`. The
 three public schemas (`verdict_json_schema`/`member_answer_json_schema`/
 `verdict_extraction_json_schema`) are a deliberate **lowest-common-denominator** shape
-(shallow nesting ≤3, enums not `oneOf`, no `$ref`, `additionalProperties:false`, optionality
-by omission) so one schema spans all three native surfaces. A **prompt-level fallback**
+(shallow nesting ≤3, enums not `oneOf`, no `$ref`, `additionalProperties:false`). The
+strict extraction schema requires every declared object property for OpenAI compatibility;
+semantic optionality uses empty arrays/strings. A **prompt-level fallback**
 (schema in messages → JSON parsed → Pydantic-validated → repair-once) is retained for
 providers without strict support; the native contract is *additive*, failure behavior
 unchanged.

src/conclave/verdict.py

@@ -57,7 +57,7 @@
 # audit can tell WHICH extractor wording produced a given clustering. Opaque
 # string; only equality/inequality is meaningful. Bump on any change to the
 # extraction system prompt in :mod:`conclave.verdict_synthesis`.
-VERDICT_EXTRACTION_PROMPT_VERSION = "3"
+VERDICT_EXTRACTION_PROMPT_VERSION = "4"
 
 
 class CouncilPosition(BaseModel):
@@ -383,16 +383,20 @@ def verdict_extraction_json_schema() -> dict:
       number; CAC-05 computes it deterministically from ``provider_votes`` via
       :mod:`conclave.agreement` (DD-1, the auditability-paradox fix).
     * **KEEPS** the judgment fields the model owns: ``verdict_type`` (enum),
-      ``headline``, ``recommendation``, ``positions``, plus the optional
-      ``conflicts``, ``provider_votes``, ``minority_reports``, ``caveats``,
-      ``dissent_summary``. ``provider_votes[*].position_label`` is what the engine
-      maps each member to in order to build the per-member clustering sequence.
+      ``headline``, ``recommendation``, ``positions``, ``conflicts``,
+      ``provider_votes``, ``minority_reports``, ``caveats``, and
+      ``dissent_summary``. Native structured-output schemas require every declared
+      object property, so semantically optional collections use an empty array and
+      optional prose uses an empty string. Optional self-reported vote confidence
+      is omitted from the extraction contract rather than forcing the model to
+      invent it. ``provider_votes[*].position_label`` is what the engine maps each
+      member to in order to build the per-member clustering sequence.
 
     Inherits all LCD constraints from the template (object nesting ≤ 3, ``enum``
     not ``oneOf``/``anyOf``/``allOf``, no ``$ref``/``$defs``,
-    ``additionalProperties: false`` on every object, optionality via
-    omission-from-required). The returned dict is a fresh object on each call so a
-    caller may mutate it freely.
+    ``additionalProperties: false`` on every object). Every declared object property
+    is required for OpenAI strict-schema compatibility. The returned dict is a
+    fresh object on each call so a caller may mutate it freely.
 
     Returns:
         A draft-style JSON Schema ``dict`` for the verdict-extraction object.
@@ -413,14 +417,35 @@ def verdict_extraction_json_schema() -> dict:
     # prompt is a decision/review at all.
     schema["properties"]["verdict_applies"] = {"type": "boolean"}
 
-    # REMOVE every consensus field — the model must never emit the number.
+    # REMOVE every top-level consensus field — the model must never emit the number.
     for field in _CONSENSUS_FIELDS:
         schema["properties"].pop(field, None)
 
-    # Rebuild ``required``: drop the consensus fields, add ``verdict_applies``
-    # (optionality stays via omission-from-required; we only add the new gate).
-    schema["required"] = [r for r in schema["required"] if r not in _CONSENSUS_FIELDS]
-    schema["required"].append("verdict_applies")
+    # Per-conflict consensus is also engine-computed. The full verdict schema
+    # carries it, but the extraction contract must never ask the model for it.
+    conflict_properties = schema["properties"]["conflicts"]["items"]["properties"]
+    conflict_properties.pop("consensus_score", None)
+
+    # Confidence is optional self-reporting and has no honest empty sentinel in
+    # its enum. Exclude it from strict extraction rather than force fabrication.
+    vote_properties = schema["properties"]["provider_votes"]["items"]["properties"]
+    vote_properties.pop("confidence", None)
+
+    # OpenAI's strict JSON-Schema surface requires every key declared in an
+    # object's ``properties`` to appear in that object's ``required`` array.
+    # Requiring empty arrays/strings for semantically optional blocks keeps this
+    # one contract valid across OpenAI, Anthropic tools, and Gemini responseSchema.
+    def require_all_declared_properties(node: object) -> None:
+        if isinstance(node, dict):
+            if node.get("type") == "object":
+                node["required"] = list(node.get("properties", {}).keys())
+            for value in node.values():
+                require_all_declared_properties(value)
+        elif isinstance(node, list):
+            for value in node:
+                require_all_declared_properties(value)
+
+    require_all_declared_properties(schema)
     return schema
 
 

src/conclave/verdict_synthesis.py

@@ -134,7 +134,9 @@ def _bounded_repair_error(detail: object) -> str:
     "answer.\n"
     "- Record one provider_vote per member that took a stance (position_label must "
     "match a positions[].label); omit a member that took no clean stance.\n"
-    "- Add conflicts[] only when there are two or more positions in tension.\n\n"
+    "- Populate conflicts[] only when there are two or more positions in tension; "
+    "otherwise emit an empty array. Emit empty arrays or an empty string for any "
+    "other schema field that does not apply.\n\n"
     "CRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement "
     "number — the council computes consensus deterministically from your "
     "clustering. Emit only the fields in the schema."

tests/evals/test_live_replay.py

@@ -29,6 +29,7 @@
 )
 from conclave.evals.runner import LIVE_HARD_CAP_USD, run_live_study
 from conclave.providers import call_model
+from conclave.verdict import verdict_extraction_json_schema
 
 FIXTURE_DIR = Path(__file__).parents[1] / "fixtures/evals/live_smoke"
 FAKE_KEY_ENV = "CONCLAVE_FAKE_TEST_KEY"
@@ -42,6 +43,34 @@ def _canonical_bytes(value: object) -> bytes:
     return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode()
 
 
+def _assert_strict_schema_instance(value: object, schema: dict, path: str = "$") -> None:
+    schema_type = schema.get("type")
+    if schema_type == "object":
+        assert isinstance(value, dict), f"{path} must be an object"
+        properties = schema["properties"]
+        assert set(schema["required"]) <= set(value), f"{path} is missing required fields"
+        if schema.get("additionalProperties") is False:
+            assert set(value) <= set(properties), f"{path} has undeclared fields"
+        for name, item in value.items():
+            _assert_strict_schema_instance(item, properties[name], f"{path}.{name}")
+    elif schema_type == "array":
+        assert isinstance(value, list), f"{path} must be an array"
+        for index, item in enumerate(value):
+            _assert_strict_schema_instance(item, schema["items"], f"{path}[{index}]")
+    elif schema_type == "string":
+        assert isinstance(value, str), f"{path} must be a string"
+    elif schema_type == "boolean":
+        assert isinstance(value, bool), f"{path} must be a boolean"
+    elif schema_type == "number":
+        assert isinstance(value, (int, float)) and not isinstance(value, bool), (
+            f"{path} must be a number"
+        )
+    else:
+        raise AssertionError(f"unsupported fixture schema type at {path}: {schema_type!r}")
+    if "enum" in schema:
+        assert value in schema["enum"], f"{path} is outside its enum"
+
+
 def _deterministic_clock():
     tick = -1
 
@@ -107,6 +136,7 @@ async def delegate(url, headers, body, timeout):
                     "conflicts": [],
                     "minority_reports": [],
                     "caveats": [],
+                    "dissent_summary": "",
                 },
                 separators=(",", ":"),
                 sort_keys=True,
@@ -178,6 +208,16 @@ async def test_live_smoke_replay_executes_all_conditions_with_zero_network_calls
     base_manifest_hash = hash_study_manifest(manifest)
     assert artifact.base_manifest_hash == base_manifest_hash
 
+    verdict_records = 0
+    for record in artifact.records:
+        messages = record.request["body"].get("messages", ())
+        if not any("verdict extractor" in message.get("content", "") for message in messages):
+            continue
+        verdict_records += 1
+        content = record.response["choices"][0]["message"]["content"]
+        _assert_strict_schema_instance(json.loads(content), verdict_extraction_json_schema())
+    assert verdict_records == len(manifest.frozen_design.rosters)
+
     delegate, delegate_calls = _fake_delegate()
     recorder = RecordingPostJson(delegate, base_manifest_hash=base_manifest_hash)
     recorded_run, recorded_checkpoint = await _execute(

tests/fixtures/evals/live_smoke/manifest.json

@@ -72,7 +72,7 @@
       "independent_synthesis": "live_synthesis_2026-06-29",
       "critique_only": "live_critique_1_synthesis_2026-06-29",
       "revision_only": "live_revision_v1_synthesis_2026-06-29",
-      "elite_full": "elite_1_synthesis_2026-06-29_verdict_3"
+      "elite_full": "elite_1_synthesis_2026-06-29_verdict_4"
     },
     "condition_protocol_versions": {
       "single_frontier": "single_frontier_live_v2",
@@ -146,11 +146,11 @@
     "preregistration_id": null,
     "preregistration_hash": null
   },
-  "frozen_design_hash": "sha256:a811130bd4a986fdacf090e66192818fa235819bc2e4f9401714e42bcb41bdac",
+  "frozen_design_hash": "sha256:3820a9f0ede9ef384f80e4a2316348c56198f13a29c043a550beaf337363f1c9",
   "planned_runs": [
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_c1a93e940605e8292b67094e",
+      "planned_run_id": "run_a3714bb73693b82cba1b46a0",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-forward",
@@ -160,7 +160,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_ed6a702b1b84a176d5f72a93",
+      "planned_run_id": "run_3959915e5861e0e386f10dd1",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-forward",
@@ -170,7 +170,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_f5fe0ed0dd5f1dae988451dd",
+      "planned_run_id": "run_e945c62092f8a1fdedda86d6",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-forward",
@@ -180,7 +180,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_bc95d6a828b012df89b302b5",
+      "planned_run_id": "run_3f4e873ac9d9cbbec606ad8b",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-forward",
@@ -190,7 +190,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_b9d0b8fe2b475b93dbf25a2c",
+      "planned_run_id": "run_8c474c7b94b16590d4d77cd2",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-forward",
@@ -200,7 +200,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_03a8814ed89c23591f147c7a",
+      "planned_run_id": "run_8d649076e8e6d213b87dfb75",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-forward",
@@ -210,7 +210,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_5658e9381ae1544529e0e864",
+      "planned_run_id": "run_640e82505d2ddd6a779005be",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-reverse",
@@ -220,7 +220,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_0bdce4dc58acd7a2e6f6a18b",
+      "planned_run_id": "run_94b3754c04fc4475df0d42da",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-reverse",
@@ -230,7 +230,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_86382226fc3cc0317ff6cbe3",
+      "planned_run_id": "run_75c738b6d6961216b51638d8",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-reverse",
@@ -240,7 +240,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_747e55eb644e03a5f1d8c156",
+      "planned_run_id": "run_c4d3105b4eacb5a97c6f2fd7",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-reverse",
@@ -250,7 +250,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_51bcfb311c31532f81ac0e5a",
+      "planned_run_id": "run_77dc81b9630663343fda69bf",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-reverse",
@@ -260,7 +260,7 @@
     },
     {
       "schema_version": "conclave_eval_v1",
-      "planned_run_id": "run_433504af776bc9a6d70694d3",
+      "planned_run_id": "run_867c14229638f374c3d091d6",
       "study_id": "fictional-live-smoke",
       "task_id": "fictional-live-task",
       "roster_id": "fictional-roster-reverse",