Skip to content

Commit ebcdfd5

Browse files
fix: adversarial mode survives a failed proposer (closes #9) (#36)
The single proposer was a single point of failure: a malformed proposer response (e.g. gemini missing candidates[0].content.parts) aborted the whole run with "no verdict produced" -- critics and judge never ran. Make the adversarial flow resilient in three layers: 1. Proposer fallback: try requested members as proposer in council order; a member returning an unusable answer (error set / no text) is recorded and the next member is tried until one produces a usable proposal. 2. Graceful degrade: if no member can propose, degrade to a plain synthesize over the survivors and surface an actionable warning on CouncilResult.synthesis_error (mirrored to adversarial.verdict_error) instead of silently emitting "no verdict produced". 3. Adapter hardening: confirmed + regression-tested that the gemini adapter turns a missing candidates[0].content.parts (or blocked/empty candidate) into a typed, redact-safe ProviderError so call_model records it as ModelAnswer.error and never raises out of the run. A failed proposer attempt is excluded from the critic set so a member is never both proposer and critic. No new result schema -- reuses existing synthesis_error / verdict_error fields. Tests: proposer fallback (default + explicit), all-proposers-fail degrade, gemini missing-parts at adapter unit level and end-to-end via call_model. Full suite 126 passed; ruff check + format clean.
1 parent 242371a commit ebcdfd5

4 files changed

Lines changed: 250 additions & 31 deletions

File tree

src/conclave/modes.py

Lines changed: 101 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -161,12 +161,26 @@ async def run_adversarial(
161161
) -> CouncilResult:
162162
"""Run a propose -> refute -> verdict pass and return a :class:`CouncilResult`.
163163
164+
The proposer is a single point of failure, so the run is layered to survive a
165+
bad one:
166+
167+
1. **Proposer fallback.** Members are tried as proposer in council order,
168+
starting with the requested one. A member that returns an unusable answer
169+
(``ModelAnswer.error`` set / no text) is recorded and the next member is
170+
tried, until one produces a usable proposal.
171+
2. **Graceful degrade.** If no member can propose, the run does *not* abort
172+
with "no verdict". It degrades to a plain synthesize over the surviving
173+
members and surfaces an actionable warning on ``CouncilResult.synthesis_error``
174+
(mirrored to the adversarial verdict so consumers reading either field see
175+
why the adversarial flow was skipped).
176+
164177
Args:
165178
council: The :class:`Council` providing fan-out, config, and judge.
166179
prompt: The user prompt.
167180
proposer: Friendly name of the proposing member. Defaults to the first
168181
requested council member. If the named proposer has no key, the run
169-
falls back to the first available member.
182+
falls back to the first available member; if its answer is unusable,
183+
the run falls back to the next available member as proposer.
170184
171185
Returns:
172186
A :class:`CouncilResult` whose ``adversarial`` field carries the proposal,
@@ -182,33 +196,48 @@ async def run_adversarial(
182196
return result
183197

184198
requested_proposer = proposer or council.requested_models[0]
185-
p_name, p_model_id = _pick_proposer(members, requested_proposer)
186-
if p_name != requested_proposer:
199+
order = _proposer_order(members, requested_proposer)
200+
201+
# Step 1: find a proposer that produces a usable answer. Each attempt is a
202+
# single-member fan-out reusing the same partial-failure primitive. Failed
203+
# attempts are recorded so the judge/degrade path can explain what was tried.
204+
base = [{"role": "user", "content": prompt}]
205+
proposal: ModelAnswer | None = None
206+
p_name = order[0][0]
207+
failed_proposers: list[ModelAnswer] = []
208+
for cand_name, cand_model_id in order:
209+
attempt = (await council.fan_out([(cand_name, cand_model_id)], lambda _n, _m: base))[0]
210+
result.answers.append(attempt)
211+
if attempt.ok:
212+
proposal = attempt
213+
p_name = cand_name
214+
break
215+
failed_proposers.append(attempt)
187216
logger.warning(
188-
"proposer '%s' unavailable; falling back to '%s'",
189-
requested_proposer,
190-
p_name,
217+
"proposer '%s' produced no usable answer (%s); trying next member",
218+
cand_name,
219+
attempt.error,
191220
)
192221

193-
# Step 1: the proposal (single-member fan-out reuses the same primitive).
194-
base = [{"role": "user", "content": prompt}]
195-
proposal = (await council.fan_out([(p_name, p_model_id)], lambda _n, _m: base))[0]
222+
# Step 2: no member could propose -> degrade to synthesize over the survivors
223+
# instead of aborting the whole run with "no verdict produced".
224+
if proposal is None:
225+
await _degrade_to_synthesize(council, result, failed_proposers)
226+
return result
196227

197228
adv = AdversarialResult(proposer=p_name, proposal=proposal)
198-
result.answers.append(proposal)
199229

200-
# Step 2: critics refute the proposal. If the proposal itself failed, there is
201-
# nothing to refute -- record the failure and skip to the (empty) judge.
202-
critics = [(n, m) for (n, m) in members if n != p_name]
203-
if proposal.ok and critics:
230+
# Step 3: critics refute the proposal. Every other available member critiques;
231+
# any failed-proposer attempts are excluded so a member is never both.
232+
tried = {a.name for a in failed_proposers} | {p_name}
233+
critics = [(n, m) for (n, m) in members if n not in tried]
234+
if critics:
204235
adv.critiques = await council.fan_out(
205236
critics, _critic_messages_for(prompt, proposal.answer or "")
206237
)
207238
result.answers.extend(adv.critiques)
208-
elif not proposal.ok:
209-
logger.warning("proposal failed (%s); critics skipped", proposal.error)
210239

211-
# Step 3: the judge weighs proposal vs critiques and issues a verdict.
240+
# Step 4: the judge weighs proposal vs critiques and issues a verdict.
212241
await _adversarial_judge(council, prompt, adv)
213242
result.adversarial = adv
214243
result.synthesis = adv.verdict
@@ -218,6 +247,48 @@ async def run_adversarial(
218247
return result
219248

220249

250+
async def _degrade_to_synthesize(
251+
council: Council,
252+
result: CouncilResult,
253+
failed_proposers: list[ModelAnswer],
254+
) -> None:
255+
"""Fall back to plain synthesize when no member can produce a proposal.
256+
257+
Every requested member was tried as proposer and none returned a usable
258+
answer, so there is nothing to refute. Rather than emit "no verdict produced",
259+
we run the standard synthesizer over whatever members *did* answer (here:
260+
none succeeded, by definition of reaching this path) and always surface an
261+
actionable warning on ``result.synthesis_error`` explaining the degrade. The
262+
warning is mirrored into an :class:`AdversarialResult` so consumers reading
263+
``result.adversarial.verdict_error`` (e.g. the CLI) see it too.
264+
"""
265+
names = ", ".join(a.name for a in failed_proposers) or "(none)"
266+
warning = (
267+
"adversarial degraded to synthesize: no council member produced a usable "
268+
f"proposal (tried: {names}). Showing a consolidated answer over the "
269+
"surviving members instead of a propose/refute/verdict result."
270+
)
271+
logger.warning(warning)
272+
273+
# Reuse the single synthesizer path. successful_answers is empty here (all
274+
# proposer attempts failed), so _synthesize records its own no-answers error;
275+
# we override synthesis_error afterwards so the degrade reason is what surfaces.
276+
await council._synthesize(result)
277+
result.synthesis_error = warning
278+
result.synthesizer = council.synthesizer
279+
result.synthesizer_model_id = council.config.resolve_model_id(council.synthesizer)
280+
281+
# Mirror the degrade into the adversarial structure so the field-specific CLI
282+
# renderer and library consumers reading result.adversarial both see it. The
283+
# first failed attempt stands in as the (failed) proposal for shape parity.
284+
if failed_proposers:
285+
adv = AdversarialResult(proposer=failed_proposers[0].name, proposal=failed_proposers[0])
286+
adv.verdict_error = warning
287+
adv.judge = council.synthesizer
288+
adv.judge_model_id = council.config.resolve_model_id(council.synthesizer)
289+
result.adversarial = adv
290+
291+
221292
def _critic_messages_for(prompt: str, proposal_text: str):
222293
"""Build the per-critic message factory for the refutation step."""
223294

@@ -230,12 +301,19 @@ def critic_messages(_name: str, _model_id: str) -> list[dict[str, str]]:
230301
return critic_messages
231302

232303

233-
def _pick_proposer(members: list[tuple[str, str]], requested: str) -> tuple[str, str]:
234-
"""Return the requested proposer member, or the first available as fallback."""
235-
for member in members:
236-
if member[0] == requested:
237-
return member
238-
return members[0]
304+
def _proposer_order(members: list[tuple[str, str]], requested: str) -> list[tuple[str, str]]:
305+
"""Return members ordered for proposer selection: requested first, then rest.
306+
307+
The requested proposer is tried first when it is available; every other
308+
available member follows in council order so a failed proposer can fall back
309+
to the next candidate. A requested name with no key is simply absent from
310+
``members`` (filtered upstream), so the council order leads.
311+
"""
312+
requested_member = next((m for m in members if m[0] == requested), None)
313+
if requested_member is None:
314+
return list(members)
315+
rest = [m for m in members if m[0] != requested]
316+
return [requested_member, *rest]
239317

240318

241319
async def _adversarial_judge(council: Council, prompt: str, adv: AdversarialResult) -> None:

tests/test_adapters.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,28 @@ def test_gemini_parse_malformed_raises():
270270
adapter.parse_response(200, {"candidates": []})
271271

272272

273+
@pytest.mark.parametrize(
274+
"payload",
275+
[
276+
# candidate present but content.parts key missing (issue #9 KeyError shape).
277+
{"candidates": [{"content": {}}]},
278+
# blocked/safety candidate with no content object at all.
279+
{"candidates": [{"finishReason": "SAFETY"}]},
280+
],
281+
)
282+
def test_gemini_parse_missing_content_parts_raises_provider_error(payload):
283+
"""A candidate missing content.parts is a typed ProviderError, never a KeyError.
284+
285+
Issue #9: a malformed/blocked Gemini response (missing
286+
``candidates[0].content.parts``) must surface as a redact-safe ProviderError
287+
so ``call_model`` can turn it into ``ModelAnswer.error`` rather than aborting
288+
the run with a raw ``KeyError``.
289+
"""
290+
adapter = GeminiAdapter()
291+
with pytest.raises(ProviderError, match="missing candidates"):
292+
adapter.parse_response(200, payload)
293+
294+
273295
def test_gemini_parse_error_status_raises():
274296
adapter = GeminiAdapter()
275297
payload = {"error": {"status": "PERMISSION_DENIED", "message": "no access"}}

tests/test_modes.py

Lines changed: 100 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -355,11 +355,18 @@ def handler(model, messages, **kwargs):
355355
assert adv.verdict == "VERDICT DESPITE FAILURE"
356356

357357

358-
async def test_adversarial_proposal_failure_skips_critics(monkeypatch, patch_call_model):
359-
"""If the proposal itself fails, critics are skipped and no verdict is made."""
358+
async def test_adversarial_all_proposers_fail_degrades_to_synthesize(monkeypatch, patch_call_model):
359+
"""If every member fails as proposer, the run degrades to synthesize (issue #9).
360+
361+
No member can produce a usable proposal, so there is nothing to refute. The
362+
run must NOT abort with "no verdict produced"; it degrades and surfaces an
363+
actionable warning on ``result.synthesis_error`` (mirrored to the adversarial
364+
``verdict_error``). Critics never run because no proposal exists.
365+
"""
360366
_all_keys(monkeypatch)
361367

362368
critic_calls: list[str] = []
369+
proposer_attempts: list[str] = []
363370

364371
def handler(model, messages, **kwargs):
365372
system = _system_text(messages)
@@ -368,21 +375,106 @@ def handler(model, messages, **kwargs):
368375
if "critic on an adversarial review" in system: # pragma: no cover
369376
critic_calls.append(model)
370377
return make_response("crit")
371-
# Proposal (grok) fails.
378+
# Every proposal attempt fails (grok first, then gemini fallback).
379+
proposer_attempts.append(model)
372380
raise RuntimeError("proposer down")
373381

374382
patch_call_model(handler)
375383

376384
council = Council(models=["grok", "gemini"], synthesizer="claude", config=_config())
377385
result = await council.adversarial("q")
378386

379-
adv = result.adversarial
380-
assert not adv.proposal.ok
381-
assert adv.critiques == []
387+
# Both members were tried as proposer in council order before degrading.
388+
assert proposer_attempts == ["xai/grok-4.3", "gemini/gemini-2.5-pro"]
389+
# No critics ran -- there was no usable proposal to refute.
382390
assert critic_calls == []
391+
# The actionable degrade warning is surfaced on the CouncilResult.
392+
assert result.synthesis_error is not None
393+
assert "degraded to synthesize" in result.synthesis_error
394+
assert "grok, gemini" in result.synthesis_error
395+
# Mirrored into the adversarial structure for field-specific consumers.
396+
adv = result.adversarial
397+
assert adv is not None
383398
assert adv.verdict is None
384-
assert "proposal from 'grok' failed" in adv.verdict_error
385-
assert result.synthesis is None
399+
assert adv.verdict_error == result.synthesis_error
400+
assert not adv.proposal.ok
401+
402+
403+
async def test_adversarial_failed_proposer_falls_back_to_next_member(monkeypatch, patch_call_model):
404+
"""A chosen proposer returning an unusable answer falls back to the next member.
405+
406+
The run completes with a real verdict (issue #9). The failed first proposer
407+
is recorded in ``answers`` but never doubles as a critic; the surviving
408+
members critique the *successful* fallback proposal.
409+
"""
410+
_all_keys(monkeypatch)
411+
412+
def handler(model, messages, **kwargs):
413+
system = _system_text(messages)
414+
if "judge of an adversarial review" in system:
415+
return make_response("VERDICT AFTER FALLBACK")
416+
if "critic on an adversarial review" in system:
417+
return make_response(f"critique from {model}")
418+
# Proposal step: the first proposer (grok) fails; gemini succeeds.
419+
if model == "xai/grok-4.3":
420+
raise RuntimeError("grok malformed proposal")
421+
return make_response(f"proposal from {model}")
422+
423+
patch_call_model(handler)
424+
425+
council = Council(
426+
models=["grok", "gemini", "perplexity"],
427+
synthesizer="claude",
428+
config=_config(),
429+
)
430+
result = await council.adversarial("q") # requested proposer = grok
431+
432+
adv = result.adversarial
433+
assert adv is not None
434+
# Fallback proposer is gemini (next available after the failed grok).
435+
assert adv.proposer == "gemini"
436+
assert adv.proposal.ok
437+
assert "proposal from gemini/gemini-2.5-pro" in adv.proposal.answer
438+
# The failed grok attempt is NOT a critic; perplexity is the sole critic.
439+
assert {c.name for c in adv.critiques} == {"perplexity"}
440+
# The run completed with a real verdict despite the proposer failure.
441+
assert adv.verdict == "VERDICT AFTER FALLBACK"
442+
assert result.synthesis == "VERDICT AFTER FALLBACK"
443+
assert result.synthesis_error is None
444+
# answers carries the failed proposal attempt + the successful proposal + critique.
445+
failed = [a for a in result.answers if not a.ok]
446+
assert {a.name for a in failed} == {"grok"}
447+
448+
449+
async def test_adversarial_explicit_failed_proposer_falls_back(monkeypatch, patch_call_model):
450+
"""An explicit --proposer that fails still falls back to the next member."""
451+
_all_keys(monkeypatch)
452+
453+
def handler(model, messages, **kwargs):
454+
system = _system_text(messages)
455+
if "judge of an adversarial review" in system:
456+
return make_response("V")
457+
if "critic on an adversarial review" in system:
458+
return make_response("crit")
459+
if model == "perplexity/sonar-pro":
460+
raise RuntimeError("perplexity proposal blocked")
461+
return make_response(f"prop {model}")
462+
463+
patch_call_model(handler)
464+
465+
council = Council(
466+
models=["grok", "gemini", "perplexity"],
467+
synthesizer="claude",
468+
config=_config(),
469+
)
470+
# Explicitly request perplexity as proposer; it fails -> first available (grok).
471+
result = await council.adversarial("q", proposer="perplexity")
472+
473+
adv = result.adversarial
474+
assert adv is not None
475+
assert adv.proposer == "grok"
476+
assert adv.proposal.ok
477+
assert adv.verdict == "V"
386478

387479

388480
async def test_adversarial_proposer_no_key_falls_back(monkeypatch, patch_call_model, clear_keys):

tests/test_providers.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,33 @@ async def boom(url, headers, json_body, timeout):
136136
assert "timed out" in answer.error
137137

138138

139+
async def test_call_model_gemini_missing_parts_becomes_error(monkeypatch):
140+
"""A malformed Gemini body (missing candidates[0].content.parts) -> ModelAnswer.error.
141+
142+
Issue #9 end-to-end: the adapter's ProviderError must flow through call_model
143+
as ``.error`` (never a raised KeyError), so one bad proposer cannot abort the
144+
adversarial run. Exercises the real GeminiAdapter + the real call_model path
145+
with only the transport patched.
146+
"""
147+
monkeypatch.setenv("GEMINI_API_KEY", "AIza-dummy")
148+
monkeypatch.setenv("CONCLAVE_CONFIG", "/nonexistent/conclave.yml")
149+
150+
async def malformed_gemini(url, headers, json_body, timeout):
151+
# 200 OK but the candidate carries no content.parts (blocked/empty shape).
152+
return 200, {"candidates": [{"finishReason": "SAFETY"}]}
153+
154+
monkeypatch.setattr("conclave.transport.post_json", malformed_gemini)
155+
156+
answer = await call_model(
157+
"gemini",
158+
"gemini/gemini-2.5-pro",
159+
[{"role": "user", "content": "hi"}],
160+
)
161+
assert not answer.ok
162+
assert answer.answer is None
163+
assert "missing candidates" in answer.error
164+
165+
139166
async def test_call_model_missing_key_is_error(monkeypatch):
140167
"""No key in env -> a clean ModelAnswer.error naming the env var, never raises."""
141168
monkeypatch.delenv("OPENAI_API_KEY", raising=False)

0 commit comments

Comments
 (0)