From 97cdfed8dea8e6e5c71f4e123d683afd61641ebb Mon Sep 17 00:00:00 2001 From: Blackwellboy Date: Sun, 26 Jul 2026 21:58:28 +1000 Subject: [PATCH] spine-probes: support thinking models in run_probes and judge run_probes.py: read thinking from message.reasoning as well as reasoning_content. Lanes exposing the former logged 0 in the reasoning column, which is the signal the README tells people to watch. judge.py: suppress thinking on the judge. max_tokens 8 is consumed by the reasoning block on a thinking model, so every verdict scored UNPARSED. --- scripts/spine-probes/judge.py | 4 ++++ scripts/spine-probes/run_probes.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/spine-probes/judge.py b/scripts/spine-probes/judge.py index 33a6553..0b422d7 100755 --- a/scripts/spine-probes/judge.py +++ b/scripts/spine-probes/judge.py @@ -46,6 +46,10 @@ def grade(url, model, token, request_text, reply, timeout): f"USER REQUEST:\n{request_text}\n\nASSISTANT REPLY:\n{reply}\n\nGrade:"}, ], "max_tokens": 8, "temperature": 0, + # A thinking judge spends all 8 tokens inside the reasoning block and + # returns empty content, so every verdict scores UNPARSED. Ask it not + # to think rather than paying for a bigger budget. + "chat_template_kwargs": {"enable_thinking": False}, }).encode() headers = {"Content-Type": "application/json"} if token: diff --git a/scripts/spine-probes/run_probes.py b/scripts/spine-probes/run_probes.py index e2d3397..24d0899 100755 --- a/scripts/spine-probes/run_probes.py +++ b/scripts/spine-probes/run_probes.py @@ -112,7 +112,10 @@ def ask(url, model, system, user, timeout, temperature, max_tokens=900): with urllib.request.urlopen(req, timeout=timeout) as r: d = json.load(r) m = d["choices"][0]["message"] - return (m.get("content") or ""), (m.get("reasoning_content") or "") + # Some OpenAI-compatible servers expose thinking on `reasoning` rather + # than `reasoning_content` (our vLLM Qwen NVFP4 lane has no + # `reasoning_content` key at all, so the reasoning column read 0). + return (m.get("content") or ""), (m.get("reasoning_content") or m.get("reasoning") or "") def main():