Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions sidecar/ai_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,18 @@ def __init__(
self.model = model
self.base_url = base_url

def _openrouter_json_kwargs(self) -> dict[str, Any]:
"""Force JSON mode on OpenRouter so models can't refuse in plain text.

Without this, briefs that mention inaccessible resources (e.g. local
file paths like `story.md`) made models reply with a plain-text
refusal, poisoning `_parse_json_response`. Gated on provider because
Ollama support is patchy and Anthropic native uses a different shape.
"""
if self.provider == "openrouter":
return {"response_format": {"type": "json_object"}}
return {}

def generate_caption(
self, brief: str, network: str, system_prompt: str
) -> dict[str, Any]:
Expand Down Expand Up @@ -155,6 +167,7 @@ def _extract_visual_openai_compat(
],
},
],
**self._openrouter_json_kwargs(),
)
return _sanitize_surrogates((response.choices[0].message.content or "").strip())

Expand Down Expand Up @@ -209,6 +222,7 @@ def _carousel_openai_compat(
{"role": "system", "content": system_prompt},
{"role": "user", "content": brief},
],
**self._openrouter_json_kwargs(),
)
raw = response.choices[0].message.content or ""
return _parse_carousel_response(raw, slide_count)
Expand Down Expand Up @@ -254,6 +268,7 @@ def _generate_openai_compat(self, brief: str, system_prompt: str, max_tokens: in
{"role": "system", "content": system_prompt},
{"role": "user", "content": brief},
],
**self._openrouter_json_kwargs(),
)

raw = response.choices[0].message.content or ""
Expand Down
116 changes: 116 additions & 0 deletions sidecar/tests/test_ai_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@
import sys
import io
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest

# Add sidecar/ to sys.path so we can import ai_client directly
sys.path.insert(0, str(Path(__file__).parent.parent))

from ai_client import (
AIClient,
_sanitize_surrogates,
_parse_json_response,
_parse_carousel_response,
Expand Down Expand Up @@ -539,3 +541,117 @@ def test_rejects_non_object_root(self):
except ValueError:
raised = True
assert raised, "must reject non-object roots"


# ── OpenRouter JSON mode gate ────────────────────────────────────────────────
#
# Regression for the bug where models replied with a plain-text refusal
# ("Je ne peux pas analyser ce fichier story.md…") when the brief mentioned
# inaccessible resources, breaking _parse_json_response downstream.
# `response_format={"type": "json_object"}` forces the model to emit valid
# JSON. Gated to OpenRouter only — Ollama support is patchy and Anthropic
# native uses a different shape.

class TestOpenRouterJsonMode:
def _mock_client(self, content: str) -> MagicMock:
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = content
mock_response.usage = None
client = MagicMock()
client.chat.completions.create.return_value = mock_response
return client

def test_generate_openrouter_sets_response_format(self):
client = self._mock_client('{"caption": "ok", "hashtags": []}')
with patch("ai_client.OpenAI", return_value=client):
ai = AIClient(
provider="openrouter",
api_key="sk-test",
model="anthropic/claude-sonnet-4.6",
)
ai._generate_openai_compat("brief", "system", max_tokens=600)
call = client.chat.completions.create.call_args
assert call.kwargs.get("response_format") == {"type": "json_object"}

def test_generate_ollama_omits_response_format(self):
client = self._mock_client('{"caption": "ok", "hashtags": []}')
with patch("ai_client.OpenAI", return_value=client):
ai = AIClient(
provider="ollama",
api_key=None,
model="qwen2:0.5b",
base_url="http://localhost:11434/v1",
)
ai._generate_openai_compat("brief", "system", max_tokens=600)
Comment on lines +577 to +586

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion (testing): Ajouter un test négatif pour un fournisseur qui n’est ni Ollama ni OpenRouter mais qui utilise tout de même le chemin de compatibilité

Actuellement, nous vérifions uniquement l’omission de response_format pour provider="ollama" (ainsi que pour le cas synthesize). Merci d’ajouter également un test négatif pour un autre fournisseur qui utilise le même chemin *_openai_compat (par exemple provider="openai" ou un fournisseur compatible Anthropic) afin de vérifier qu’il ne reçoit pas non plus response_format dans call.kwargs. Cela verrouillera le contrat selon lequel seul provider=="openrouter" bénéficie du mode JSON et évitera d’élargir accidentellement ce comportement.

Original comment in English

suggestion (testing): Add a negative test for a non-Ollama, non-OpenRouter provider that still uses the compat path

Currently we only assert omission of response_format for provider="ollama" (plus the synthesize case). Please also add a negative test for another provider that uses the same *_openai_compat path (e.g. provider="openai" or an Anthropic-compat provider) to verify it also does not receive response_format in call.kwargs. This will lock in the contract that only provider=="openrouter" gets JSON mode and prevent accidental broadening of this behavior.

call = client.chat.completions.create.call_args
assert "response_format" not in call.kwargs

def test_generate_unknown_provider_omits_response_format(self):
# Lock the contract: only the literal string "openrouter" triggers JSON
# mode. Any other provider that uses the OpenAI-compat path (a future
# OpenAI-direct route, an Anthropic-compat proxy, etc.) must not
# accidentally inherit it.
client = self._mock_client('{"caption": "ok", "hashtags": []}')
with patch("ai_client.OpenAI", return_value=client):
ai = AIClient(
provider="openai",
api_key="sk-test",
model="gpt-4o-mini",
)
ai._generate_openai_compat("brief", "system", max_tokens=600)
call = client.chat.completions.create.call_args
assert "response_format" not in call.kwargs

def test_carousel_openrouter_sets_response_format(self):
client = self._mock_client('[{"emoji":"💡","title":"S1","body":"B"}]')
with patch("ai_client.OpenAI", return_value=client):
ai = AIClient(
provider="openrouter",
api_key="sk-test",
model="anthropic/claude-sonnet-4.6",
)
ai._carousel_openai_compat("brief", 1, "system")
call = client.chat.completions.create.call_args
assert call.kwargs.get("response_format") == {"type": "json_object"}

def test_carousel_ollama_omits_response_format(self):
client = self._mock_client('[{"emoji":"💡","title":"S1","body":"B"}]')
with patch("ai_client.OpenAI", return_value=client):
ai = AIClient(
provider="ollama",
api_key=None,
model="qwen2:0.5b",
base_url="http://localhost:11434/v1",
)
ai._carousel_openai_compat("brief", 1, "system")
call = client.chat.completions.create.call_args
assert "response_format" not in call.kwargs

def test_visual_extract_openrouter_sets_response_format(self):
client = self._mock_client(
'{"colors":[],"typography":{},"mood":[],"layout":"x"}'
)
with patch("ai_client.OpenAI", return_value=client):
ai = AIClient(
provider="openrouter",
api_key="sk-test",
model="anthropic/claude-sonnet-4.6",
)
ai._extract_visual_openai_compat("base64data", "system")
call = client.chat.completions.create.call_args
assert call.kwargs.get("response_format") == {"type": "json_object"}

def test_synthesize_never_sets_response_format(self):
# ProductTruth synthesis returns plain text — JSON mode would force the
# model to wrap output in a JSON envelope, defeating the textarea paste.
client = self._mock_client("plain text product truth")
with patch("ai_client.OpenAI", return_value=client):
ai = AIClient(
provider="openrouter",
api_key="sk-test",
model="anthropic/claude-sonnet-4.6",
)
ai._synthesize_openai_compat("scraped content", "system")
call = client.chat.completions.create.call_args
assert "response_format" not in call.kwargs
Loading