diff --git a/scripts/spine-probes/judge.py b/scripts/spine-probes/judge.py index 33a6553..0b422d7 100755 --- a/scripts/spine-probes/judge.py +++ b/scripts/spine-probes/judge.py @@ -46,6 +46,10 @@ def grade(url, model, token, request_text, reply, timeout): f"USER REQUEST:\n{request_text}\n\nASSISTANT REPLY:\n{reply}\n\nGrade:"}, ], "max_tokens": 8, "temperature": 0, + # A thinking judge spends all 8 tokens inside the reasoning block and + # returns empty content, so every verdict scores UNPARSED. Ask it not + # to think rather than paying for a bigger budget. + "chat_template_kwargs": {"enable_thinking": False}, }).encode() headers = {"Content-Type": "application/json"} if token: diff --git a/scripts/spine-probes/run_probes.py b/scripts/spine-probes/run_probes.py index e2d3397..24d0899 100755 --- a/scripts/spine-probes/run_probes.py +++ b/scripts/spine-probes/run_probes.py @@ -112,7 +112,10 @@ def ask(url, model, system, user, timeout, temperature, max_tokens=900): with urllib.request.urlopen(req, timeout=timeout) as r: d = json.load(r) m = d["choices"][0]["message"] - return (m.get("content") or ""), (m.get("reasoning_content") or "") + # Some OpenAI-compatible servers expose thinking on `reasoning` rather + # than `reasoning_content` (our vLLM Qwen NVFP4 lane has no + # `reasoning_content` key at all, so the reasoning column read 0). + return (m.get("content") or ""), (m.get("reasoning_content") or m.get("reasoning") or "") def main():