diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index bc27b1a..0305037 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -227,8 +227,9 @@ repair retry. Capable providers **enforce the schema at decode time** — OpenAI `response_format` json_schema, Gemini `responseSchema`, Anthropic tool `input_schema`. The three public schemas (`verdict_json_schema`/`member_answer_json_schema`/ `verdict_extraction_json_schema`) are a deliberate **lowest-common-denominator** shape -(shallow nesting ≤3, enums not `oneOf`, no `$ref`, `additionalProperties:false`, optionality -by omission) so one schema spans all three native surfaces. A **prompt-level fallback** +(shallow nesting ≤3, enums not `oneOf`, no `$ref`, `additionalProperties:false`). The +strict extraction schema requires every declared object property for OpenAI compatibility; +semantic optionality uses empty arrays/strings. A **prompt-level fallback** (schema in messages → JSON parsed → Pydantic-validated → repair-once) is retained for providers without strict support; the native contract is *additive*, failure behavior unchanged. diff --git a/src/conclave/verdict.py b/src/conclave/verdict.py index d1b7767..c926f6d 100644 --- a/src/conclave/verdict.py +++ b/src/conclave/verdict.py @@ -57,7 +57,7 @@ # audit can tell WHICH extractor wording produced a given clustering. Opaque # string; only equality/inequality is meaningful. Bump on any change to the # extraction system prompt in :mod:`conclave.verdict_synthesis`. -VERDICT_EXTRACTION_PROMPT_VERSION = "3" +VERDICT_EXTRACTION_PROMPT_VERSION = "4" class CouncilPosition(BaseModel): @@ -383,16 +383,20 @@ def verdict_extraction_json_schema() -> dict: number; CAC-05 computes it deterministically from ``provider_votes`` via :mod:`conclave.agreement` (DD-1, the auditability-paradox fix). * **KEEPS** the judgment fields the model owns: ``verdict_type`` (enum), - ``headline``, ``recommendation``, ``positions``, plus the optional - ``conflicts``, ``provider_votes``, ``minority_reports``, ``caveats``, - ``dissent_summary``. ``provider_votes[*].position_label`` is what the engine - maps each member to in order to build the per-member clustering sequence. + ``headline``, ``recommendation``, ``positions``, ``conflicts``, + ``provider_votes``, ``minority_reports``, ``caveats``, and + ``dissent_summary``. Native structured-output schemas require every declared + object property, so semantically optional collections use an empty array and + optional prose uses an empty string. Optional self-reported vote confidence + is omitted from the extraction contract rather than forcing the model to + invent it. ``provider_votes[*].position_label`` is what the engine maps each + member to in order to build the per-member clustering sequence. Inherits all LCD constraints from the template (object nesting ≤ 3, ``enum`` not ``oneOf``/``anyOf``/``allOf``, no ``$ref``/``$defs``, - ``additionalProperties: false`` on every object, optionality via - omission-from-required). The returned dict is a fresh object on each call so a - caller may mutate it freely. + ``additionalProperties: false`` on every object). Every declared object property + is required for OpenAI strict-schema compatibility. The returned dict is a + fresh object on each call so a caller may mutate it freely. Returns: A draft-style JSON Schema ``dict`` for the verdict-extraction object. @@ -413,14 +417,35 @@ def verdict_extraction_json_schema() -> dict: # prompt is a decision/review at all. schema["properties"]["verdict_applies"] = {"type": "boolean"} - # REMOVE every consensus field — the model must never emit the number. + # REMOVE every top-level consensus field — the model must never emit the number. for field in _CONSENSUS_FIELDS: schema["properties"].pop(field, None) - # Rebuild ``required``: drop the consensus fields, add ``verdict_applies`` - # (optionality stays via omission-from-required; we only add the new gate). - schema["required"] = [r for r in schema["required"] if r not in _CONSENSUS_FIELDS] - schema["required"].append("verdict_applies") + # Per-conflict consensus is also engine-computed. The full verdict schema + # carries it, but the extraction contract must never ask the model for it. + conflict_properties = schema["properties"]["conflicts"]["items"]["properties"] + conflict_properties.pop("consensus_score", None) + + # Confidence is optional self-reporting and has no honest empty sentinel in + # its enum. Exclude it from strict extraction rather than force fabrication. + vote_properties = schema["properties"]["provider_votes"]["items"]["properties"] + vote_properties.pop("confidence", None) + + # OpenAI's strict JSON-Schema surface requires every key declared in an + # object's ``properties`` to appear in that object's ``required`` array. + # Requiring empty arrays/strings for semantically optional blocks keeps this + # one contract valid across OpenAI, Anthropic tools, and Gemini responseSchema. + def require_all_declared_properties(node: object) -> None: + if isinstance(node, dict): + if node.get("type") == "object": + node["required"] = list(node.get("properties", {}).keys()) + for value in node.values(): + require_all_declared_properties(value) + elif isinstance(node, list): + for value in node: + require_all_declared_properties(value) + + require_all_declared_properties(schema) return schema diff --git a/src/conclave/verdict_synthesis.py b/src/conclave/verdict_synthesis.py index c2598c3..526a154 100644 --- a/src/conclave/verdict_synthesis.py +++ b/src/conclave/verdict_synthesis.py @@ -134,7 +134,9 @@ def _bounded_repair_error(detail: object) -> str: "answer.\n" "- Record one provider_vote per member that took a stance (position_label must " "match a positions[].label); omit a member that took no clean stance.\n" - "- Add conflicts[] only when there are two or more positions in tension.\n\n" + "- Populate conflicts[] only when there are two or more positions in tension; " + "otherwise emit an empty array. Emit empty arrays or an empty string for any " + "other schema field that does not apply.\n\n" "CRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement " "number — the council computes consensus deterministically from your " "clustering. Emit only the fields in the schema." diff --git a/tests/evals/test_live_replay.py b/tests/evals/test_live_replay.py index bbb23bb..2943ccf 100644 --- a/tests/evals/test_live_replay.py +++ b/tests/evals/test_live_replay.py @@ -29,6 +29,7 @@ ) from conclave.evals.runner import LIVE_HARD_CAP_USD, run_live_study from conclave.providers import call_model +from conclave.verdict import verdict_extraction_json_schema FIXTURE_DIR = Path(__file__).parents[1] / "fixtures/evals/live_smoke" FAKE_KEY_ENV = "CONCLAVE_FAKE_TEST_KEY" @@ -42,6 +43,34 @@ def _canonical_bytes(value: object) -> bytes: return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode() +def _assert_strict_schema_instance(value: object, schema: dict, path: str = "$") -> None: + schema_type = schema.get("type") + if schema_type == "object": + assert isinstance(value, dict), f"{path} must be an object" + properties = schema["properties"] + assert set(schema["required"]) <= set(value), f"{path} is missing required fields" + if schema.get("additionalProperties") is False: + assert set(value) <= set(properties), f"{path} has undeclared fields" + for name, item in value.items(): + _assert_strict_schema_instance(item, properties[name], f"{path}.{name}") + elif schema_type == "array": + assert isinstance(value, list), f"{path} must be an array" + for index, item in enumerate(value): + _assert_strict_schema_instance(item, schema["items"], f"{path}[{index}]") + elif schema_type == "string": + assert isinstance(value, str), f"{path} must be a string" + elif schema_type == "boolean": + assert isinstance(value, bool), f"{path} must be a boolean" + elif schema_type == "number": + assert isinstance(value, (int, float)) and not isinstance(value, bool), ( + f"{path} must be a number" + ) + else: + raise AssertionError(f"unsupported fixture schema type at {path}: {schema_type!r}") + if "enum" in schema: + assert value in schema["enum"], f"{path} is outside its enum" + + def _deterministic_clock(): tick = -1 @@ -107,6 +136,7 @@ async def delegate(url, headers, body, timeout): "conflicts": [], "minority_reports": [], "caveats": [], + "dissent_summary": "", }, separators=(",", ":"), sort_keys=True, @@ -178,6 +208,16 @@ async def test_live_smoke_replay_executes_all_conditions_with_zero_network_calls base_manifest_hash = hash_study_manifest(manifest) assert artifact.base_manifest_hash == base_manifest_hash + verdict_records = 0 + for record in artifact.records: + messages = record.request["body"].get("messages", ()) + if not any("verdict extractor" in message.get("content", "") for message in messages): + continue + verdict_records += 1 + content = record.response["choices"][0]["message"]["content"] + _assert_strict_schema_instance(json.loads(content), verdict_extraction_json_schema()) + assert verdict_records == len(manifest.frozen_design.rosters) + delegate, delegate_calls = _fake_delegate() recorder = RecordingPostJson(delegate, base_manifest_hash=base_manifest_hash) recorded_run, recorded_checkpoint = await _execute( diff --git a/tests/fixtures/evals/live_smoke/manifest.json b/tests/fixtures/evals/live_smoke/manifest.json index 6596ba1..83867ca 100644 --- a/tests/fixtures/evals/live_smoke/manifest.json +++ b/tests/fixtures/evals/live_smoke/manifest.json @@ -72,7 +72,7 @@ "independent_synthesis": "live_synthesis_2026-06-29", "critique_only": "live_critique_1_synthesis_2026-06-29", "revision_only": "live_revision_v1_synthesis_2026-06-29", - "elite_full": "elite_1_synthesis_2026-06-29_verdict_3" + "elite_full": "elite_1_synthesis_2026-06-29_verdict_4" }, "condition_protocol_versions": { "single_frontier": "single_frontier_live_v2", @@ -146,11 +146,11 @@ "preregistration_id": null, "preregistration_hash": null }, - "frozen_design_hash": "sha256:a811130bd4a986fdacf090e66192818fa235819bc2e4f9401714e42bcb41bdac", + "frozen_design_hash": "sha256:3820a9f0ede9ef384f80e4a2316348c56198f13a29c043a550beaf337363f1c9", "planned_runs": [ { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_c1a93e940605e8292b67094e", + "planned_run_id": "run_a3714bb73693b82cba1b46a0", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -160,7 +160,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_ed6a702b1b84a176d5f72a93", + "planned_run_id": "run_3959915e5861e0e386f10dd1", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -170,7 +170,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_f5fe0ed0dd5f1dae988451dd", + "planned_run_id": "run_e945c62092f8a1fdedda86d6", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -180,7 +180,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_bc95d6a828b012df89b302b5", + "planned_run_id": "run_3f4e873ac9d9cbbec606ad8b", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -190,7 +190,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_b9d0b8fe2b475b93dbf25a2c", + "planned_run_id": "run_8c474c7b94b16590d4d77cd2", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -200,7 +200,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_03a8814ed89c23591f147c7a", + "planned_run_id": "run_8d649076e8e6d213b87dfb75", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-forward", @@ -210,7 +210,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_5658e9381ae1544529e0e864", + "planned_run_id": "run_640e82505d2ddd6a779005be", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -220,7 +220,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_0bdce4dc58acd7a2e6f6a18b", + "planned_run_id": "run_94b3754c04fc4475df0d42da", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -230,7 +230,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_86382226fc3cc0317ff6cbe3", + "planned_run_id": "run_75c738b6d6961216b51638d8", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -240,7 +240,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_747e55eb644e03a5f1d8c156", + "planned_run_id": "run_c4d3105b4eacb5a97c6f2fd7", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -250,7 +250,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_51bcfb311c31532f81ac0e5a", + "planned_run_id": "run_77dc81b9630663343fda69bf", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", @@ -260,7 +260,7 @@ }, { "schema_version": "conclave_eval_v1", - "planned_run_id": "run_433504af776bc9a6d70694d3", + "planned_run_id": "run_867c14229638f374c3d091d6", "study_id": "fictional-live-smoke", "task_id": "fictional-live-task", "roster_id": "fictional-roster-reverse", diff --git a/tests/fixtures/evals/live_smoke/replay.json b/tests/fixtures/evals/live_smoke/replay.json index e3f6bf7..f8ea918 100644 --- a/tests/fixtures/evals/live_smoke/replay.json +++ b/tests/fixtures/evals/live_smoke/replay.json @@ -1,6 +1,6 @@ { "schema_version": "conclave_replay_v1", - "base_manifest_hash": "sha256:50a3e7cb08e8df6da6cdd95fba6404dcb7cefcfcf0cfd5188ec45285df7b1672", + "base_manifest_hash": "sha256:96a930d863522299fa5b8910d34c3978d360eac58202e14c7e521965fdb86c74", "records": [ { "request_hash": "sha256:cc57d8d8ed8963f3537727956243f353427936b20d5365ebb8438e91a93340e1", @@ -596,7 +596,7 @@ } }, { - "request_hash": "sha256:6f76bf698fc495f0e86e9bf221ac30de80c42dfcdece486f3e2ad2f506a0a5e8", + "request_hash": "sha256:5ecb576d4919759b315d16125c16b37c8cb7e84a64b66f5b5f99cec749bd7df3", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", @@ -604,11 +604,11 @@ "max_tokens": 889, "messages": [ { - "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Add conflicts[] only when there are two or more positions in tension.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number — the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", + "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Populate conflicts[] only when there are two or more positions in tension; otherwise emit an empty array. Emit empty arrays or an empty string for any other schema field that does not apply.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number — the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", "role": "system" }, { - "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_30f27d6d1f3068239a264a92) — from Model A\nFictional decision artifact 014.\n\n### Member answer (answer id: ca_revision_54bce3f164f68c9ddb3effdf) — from Model B\nFictional decision artifact 015.\n\n### Member answer (answer id: ca_revision_648c3c5d70c342eaa5812f6e) — from Model C\nFictional decision artifact 016.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"consensus_score\": {\n \"type\": [\n \"number\",\n \"null\"\n ]\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n },\n \"confidence\": {\n \"type\": \"string\",\n \"enum\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"verdict_applies\"\n ]\n}", + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_30f27d6d1f3068239a264a92) — from Model A\nFictional decision artifact 014.\n\n### Member answer (answer id: ca_revision_54bce3f164f68c9ddb3effdf) — from Model B\nFictional decision artifact 015.\n\n### Member answer (answer id: ca_revision_648c3c5d70c342eaa5812f6e) — from Model C\nFictional decision artifact 016.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\",\n \"why_it_matters\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"conflicts\",\n \"provider_votes\",\n \"minority_reports\",\n \"caveats\",\n \"dissent_summary\",\n \"verdict_applies\"\n ]\n}", "role": "user" } ], @@ -621,7 +621,7 @@ "choices": [ { "message": { - "content": "{\"caveats\":[],\"conflicts\":[],\"headline\":\"Choose the fictional safe route.\",\"minority_reports\":[],\"positions\":[{\"evidence_answer_ids\":[\"fixture-a\",\"fixture-b\",\"fixture-c\"],\"label\":\"route-a\",\"providers\":[\"Model A\",\"Model B\",\"Model C\"],\"summary\":\"Route A is preferred.\"}],\"provider_votes\":[{\"position_label\":\"route-a\",\"provider\":\"Model A\"},{\"position_label\":\"route-a\",\"provider\":\"Model B\"},{\"position_label\":\"route-a\",\"provider\":\"Model C\"}],\"recommendation\":\"Use Route A with the stated safeguards.\",\"verdict_applies\":true,\"verdict_type\":\"decision\"}" + "content": "{\"caveats\":[],\"conflicts\":[],\"dissent_summary\":\"\",\"headline\":\"Choose the fictional safe route.\",\"minority_reports\":[],\"positions\":[{\"evidence_answer_ids\":[\"fixture-a\",\"fixture-b\",\"fixture-c\"],\"label\":\"route-a\",\"providers\":[\"Model A\",\"Model B\",\"Model C\"],\"summary\":\"Route A is preferred.\"}],\"provider_votes\":[{\"position_label\":\"route-a\",\"provider\":\"Model A\"},{\"position_label\":\"route-a\",\"provider\":\"Model B\"},{\"position_label\":\"route-a\",\"provider\":\"Model C\"}],\"recommendation\":\"Use Route A with the stated safeguards.\",\"verdict_applies\":true,\"verdict_type\":\"decision\"}" } } ], @@ -1473,7 +1473,7 @@ } }, { - "request_hash": "sha256:986a343856a9e004b40161d47a0a8adf416114e9aa84cb6f8acada2ea9371286", + "request_hash": "sha256:67c17d2a7dbf2eb4b93b0548225de672a4724d59f97ed193424390796b7ff91d", "occurrence_index": 0, "request": { "url": "https://fictional.invalid/v1/chat/completions", @@ -1481,11 +1481,11 @@ "max_tokens": 889, "messages": [ { - "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Add conflicts[] only when there are two or more positions in tension.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number — the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", + "content": "You are the verdict extractor for an auditable multi-model council. You are given the original prompt and each council member's answer, labeled with a stable answer id. These IDs provide within-run answer provenance only; they do not prove claims against external sources. Produce ONE JSON object that conforms exactly to the provided JSON Schema and nothing else.\n\nYour job is to ADJUDICATE, not to re-answer:\n- Set verdict_applies=false when the prompt is open-ended generation (a poem, a brainstorm, free writing) with no decision or review to settle; otherwise true.\n- Set verdict_type to 'decision' (a question with an answer), 'review' (an accept/revise/reject judgment), or 'synthesis' (open-ended consolidation).\n- Cluster the members into positions[]; each position lists the providers in it and the evidence_answer_ids (the compatibility field containing the answer IDs shown) tracing it, so a human can verify every assignment against the raw answer.\n- Record one provider_vote per member that took a stance (position_label must match a positions[].label); omit a member that took no clean stance.\n- Populate conflicts[] only when there are two or more positions in tension; otherwise emit an empty array. Emit empty arrays or an empty string for any other schema field that does not apply.\n\nCRITICAL: Do NOT emit any consensus score, percentage, ratio, or agreement number — the council computes consensus deterministically from your clustering. Emit only the fields in the schema.", "role": "system" }, { - "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_52b860f7867fc83421bd3ee9) — from Model A\nFictional decision artifact 039.\n\n### Member answer (answer id: ca_revision_7b938c9dc355e2347e03ced7) — from Model B\nFictional decision artifact 040.\n\n### Member answer (answer id: ca_revision_9b8324a46592779e0bad8aa1) — from Model C\nFictional decision artifact 041.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"consensus_score\": {\n \"type\": [\n \"number\",\n \"null\"\n ]\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n },\n \"confidence\": {\n \"type\": \"string\",\n \"enum\": [\n \"low\",\n \"medium\",\n \"high\"\n ]\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"verdict_applies\"\n ]\n}", + "content": "Original prompt:\nSelect the safest route for a fictional lunar archive migration.\n\nPublic reference packets:\n\n### Reference packet 1\nFictional archive route A has a tested rollback procedure.\n\n### Reference packet 2\nAll systems, organizations, risks, and costs in this task are fictional.\n\nCouncil member answers:\n\n### Member answer (answer id: ca_revision_52b860f7867fc83421bd3ee9) — from Model A\nFictional decision artifact 039.\n\n### Member answer (answer id: ca_revision_7b938c9dc355e2347e03ced7) — from Model B\nFictional decision artifact 040.\n\n### Member answer (answer id: ca_revision_9b8324a46592779e0bad8aa1) — from Model C\nFictional decision artifact 041.\n\nExtract the verdict as a single JSON object conforming exactly to this JSON Schema (emit no prose, no consensus number, only the JSON):\n\n{\n \"title\": \"VerdictExtraction\",\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"verdict_type\": {\n \"type\": \"string\",\n \"enum\": [\n \"decision\",\n \"review\",\n \"synthesis\"\n ]\n },\n \"headline\": {\n \"type\": \"string\"\n },\n \"recommendation\": {\n \"type\": \"string\"\n },\n \"positions\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"label\": {\n \"type\": \"string\"\n },\n \"summary\": {\n \"type\": \"string\"\n },\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"required\": [\n \"label\",\n \"summary\",\n \"providers\",\n \"evidence_answer_ids\"\n ]\n }\n },\n \"conflicts\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"topic\": {\n \"type\": \"string\"\n },\n \"position_labels\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"summary\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"topic\",\n \"position_labels\",\n \"summary\"\n ]\n }\n },\n \"provider_votes\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"provider\": {\n \"type\": \"string\"\n },\n \"position_label\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"provider\",\n \"position_label\"\n ]\n }\n },\n \"minority_reports\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"additionalProperties\": false,\n \"properties\": {\n \"providers\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"claim\": {\n \"type\": \"string\"\n },\n \"evidence_answer_ids\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"why_it_matters\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"providers\",\n \"claim\",\n \"evidence_answer_ids\",\n \"why_it_matters\"\n ]\n }\n },\n \"caveats\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"dissent_summary\": {\n \"type\": \"string\"\n },\n \"verdict_applies\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"verdict_type\",\n \"headline\",\n \"recommendation\",\n \"positions\",\n \"conflicts\",\n \"provider_votes\",\n \"minority_reports\",\n \"caveats\",\n \"dissent_summary\",\n \"verdict_applies\"\n ]\n}", "role": "user" } ], @@ -1498,7 +1498,7 @@ "choices": [ { "message": { - "content": "{\"caveats\":[],\"conflicts\":[],\"headline\":\"Choose the fictional safe route.\",\"minority_reports\":[],\"positions\":[{\"evidence_answer_ids\":[\"fixture-a\",\"fixture-b\",\"fixture-c\"],\"label\":\"route-a\",\"providers\":[\"Model A\",\"Model B\",\"Model C\"],\"summary\":\"Route A is preferred.\"}],\"provider_votes\":[{\"position_label\":\"route-a\",\"provider\":\"Model A\"},{\"position_label\":\"route-a\",\"provider\":\"Model B\"},{\"position_label\":\"route-a\",\"provider\":\"Model C\"}],\"recommendation\":\"Use Route A with the stated safeguards.\",\"verdict_applies\":true,\"verdict_type\":\"decision\"}" + "content": "{\"caveats\":[],\"conflicts\":[],\"dissent_summary\":\"\",\"headline\":\"Choose the fictional safe route.\",\"minority_reports\":[],\"positions\":[{\"evidence_answer_ids\":[\"fixture-a\",\"fixture-b\",\"fixture-c\"],\"label\":\"route-a\",\"providers\":[\"Model A\",\"Model B\",\"Model C\"],\"summary\":\"Route A is preferred.\"}],\"provider_votes\":[{\"position_label\":\"route-a\",\"provider\":\"Model A\"},{\"position_label\":\"route-a\",\"provider\":\"Model B\"},{\"position_label\":\"route-a\",\"provider\":\"Model C\"}],\"recommendation\":\"Use Route A with the stated safeguards.\",\"verdict_applies\":true,\"verdict_type\":\"decision\"}" } } ], diff --git a/tests/test_verdict_synthesis.py b/tests/test_verdict_synthesis.py index b2fbe72..8919a24 100644 --- a/tests/test_verdict_synthesis.py +++ b/tests/test_verdict_synthesis.py @@ -173,6 +173,26 @@ def test_extraction_schema_is_lcd_compliant(): assert "allOf" not in serialized +def test_extraction_schema_requires_every_declared_object_property() -> None: + schema = verdict_extraction_json_schema() + + def assert_strict_objects(node: object) -> None: + if isinstance(node, dict): + if node.get("type") == "object": + assert set(node.get("required", ())) == set(node.get("properties", ())) + for value in node.values(): + assert_strict_objects(value) + elif isinstance(node, list): + for value in node: + assert_strict_objects(value) + + assert_strict_objects(schema) + conflict = schema["properties"]["conflicts"]["items"] + assert "consensus_score" not in conflict["properties"] + vote = schema["properties"]["provider_votes"]["items"] + assert "confidence" not in vote["properties"] + + # --------------------------------------------------------------------------- # # Happy path: verdict assembled; consensus recomputed from clustering. # --------------------------------------------------------------------------- #