From df21549cae082e484697422cc7c24eec15439dd1 Mon Sep 17 00:00:00 2001 From: Christopher Maher Date: Sat, 25 Jul 2026 21:39:42 -0700 Subject: [PATCH] scripts: add thinking probes for measuring what controls reasoning Companion to spine-probes, different question: whether a model actually reasons, and what controls it. render_check.py compares what the chat template renders per kwarg value via /apply-template, deterministic and needing no statistics, and optionally diffs that against a local jinja2 render of the same file to show when the server overrides the template thinking_ab.py interleaved sampling across the same arms, classifying reasoned-and-parsed vs reasoned-but-unparsed vs not-reasoned persona_ab.py crosses a persona system prompt against the kwarg, with the two structurally-impossible cells as a control These come out of getting a result wrong (#5). We reported the enable_thinking kwarg as inert on Laguna S 2.1, reasoning from the template source, which sets default(false). The template reading was correct; the conclusion was not, because the server supplies that kwarg itself when the client omits it, so the default is unreachable. Two habits are baked in as a result. Reading is not measuring, so render_check inspects rendered output rather than source. And a finding needs a cell where it cannot happen, so both sampling tools carry a control arm. Both sampling tools interleave arms, vary prompt shape, disable the prompt cache with a unique nonce per request, and hold max_tokens fixed with no retry. Our first pass did none of that and produced 5/5 against 0/5 that was mostly artifact. Manufactured consistency is indistinguishable from a strong result by inspection. Signed-off-by: Christopher Maher --- scripts/thinking-probes/README.md | 105 +++++++++++++++ scripts/thinking-probes/persona_ab.py | 147 +++++++++++++++++++++ scripts/thinking-probes/render_check.py | 122 +++++++++++++++++ scripts/thinking-probes/thinking_ab.py | 169 ++++++++++++++++++++++++ 4 files changed, 543 insertions(+) create mode 100644 scripts/thinking-probes/README.md create mode 100755 scripts/thinking-probes/persona_ab.py create mode 100644 scripts/thinking-probes/render_check.py create mode 100755 scripts/thinking-probes/thinking_ab.py diff --git a/scripts/thinking-probes/README.md b/scripts/thinking-probes/README.md new file mode 100644 index 0000000..1339bc5 --- /dev/null +++ b/scripts/thinking-probes/README.md @@ -0,0 +1,105 @@ +# Thinking probes + +Three tools for measuring **whether a model actually reasons**, and what +controls it, on an OpenAI-compatible endpoint. + +Separate from `spine-probes/`, which measures integrity behaviour. Different +question, same house rules: standard library only, no install step, every raw +response saved. + +These exist because of a wrong result. We reported that the `enable_thinking` +chat-template kwarg was inert on Laguna S 2.1, reasoning from the template +source, which sets `| default(false)`. The template was read correctly. The +conclusion was still wrong, because the server supplies that kwarg itself when +the client omits it, so the default never runs. See +[#5](https://github.com/TheTom/offlabel/issues/5) for the full correction. + +Two habits came out of that and are baked into these tools. + +**Reading is not measuring.** `render_check.py` inspects rendered output rather +than template source, and needs no statistics to do it. + +**A finding needs a cell where it cannot happen.** Both sampling tools include a +control arm where the effect is structurally impossible. If the effect shows up +there anyway, the instrument is broken and every other number is suspect. + +## `render_check.py`: what the template actually renders + +The cheapest and most decisive check here. `/apply-template` returns the +formatted prompt without generating, so it is deterministic and repeats exactly. + +```bash +python3 render_check.py --base http://localhost:8080 +``` + +It varies one kwarg across four arms (absent, empty object, `false`, `true`) and +reports which arms render identically. **Test the absent arm.** It is the one +most people never check and the one most people actually run. + +Point it at the template file as well and it renders the same file in stock +jinja2 and compares: + +```bash +python3 render_check.py --base http://localhost:8080 \ + --template models/templates/poolside-Laguna-S-2.1.jinja +``` + +An arm reported as `SERVER OVERRIDES TEMPLATE` means the server is supplying +that kwarg itself. In that case the template's default for it is unreachable, +and reasoning about behaviour from the template alone will mislead you. That is +exactly the trap we fell into. + +Use `--kwarg` for anything other than `enable_thinking`. + +## `thinking_ab.py`: does the rendering change behaviour + +Interleaved sampling across the same arms, to confirm the rendering has the +consequence it looks like it should. + +```bash +python3 thinking_ab.py --base http://localhost:8080 --model my-model --seeds 5 +``` + +Classifies each response three ways, which is the distinction a single field +cannot make: + +| signal | verdict | +|---|---| +| `reasoning_content` non-empty | `REASONED_PARSED` | +| `` markers in content | `REASONED_UNPARSED` | +| neither | `NO_REASONING` | + +`REASONED_UNPARSED` is the important one. Any occurrence means a stack reading +only `reasoning_content` would report that arm as not firing when it did, so a +published firing rate from such a stack is measuring the parser as much as the +model. + +## `persona_ab.py`: does a system prompt change it + +Crosses a persona system prompt against the kwarg, four cells: + +```bash +python3 persona_ab.py --base http://localhost:8080 --model my-model --seeds 6 +``` + +The two `false` cells are the control. Thinking is structurally unavailable +there, so the persona must show no effect. If it appears to, stop and fix the +measurement before believing the other two cells. + +## Design notes, and one warning + +Both sampling tools interleave arms round robin, vary the prompt across three +shapes, send `cache_prompt: false` with a unique nonce per request, and hold +`max_tokens` fixed across every arm with no retry. + +That is not fussiness. Our first pass ran arms in blocks, on one repeated +prompt, with the cache on, and produced 5/5 in one arm against 0/5 in another. +It was mostly an artifact. **Manufactured consistency looks exactly like a +strong result**, and no amount of staring at the numbers reveals it. + +Vary the token budget across arms and you have the same problem in another form: +a difference in outcomes that cannot be attributed to the variable you meant to +test. + +Task shape moves these numbers a lot, so the per-shape breakdown in the output +is worth reading before drawing a conclusion from the pooled figure. diff --git a/scripts/thinking-probes/persona_ab.py b/scripts/thinking-probes/persona_ab.py new file mode 100755 index 0000000..4838104 --- /dev/null +++ b/scripts/thinking-probes/persona_ab.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Re-measure the persona effect against a baseline we have actually measured. + +The original persona result (6/6 fired without a persona, 0/5 with) was taken +in the enable_thinking-absent condition, on one prompt, in blocks. We since +established that the absent condition reasons only about half the time and +that task shape moves it a lot, so the old baseline was not the near-certainty +it looked like. The effect may well be real; the effect SIZE was measured +against a constant that is not constant. + +This crosses the two factors instead of holding one at an assumed value: + + persona in {none, senior-staff-engineer} + kwarg in {absent, false} + +The false rows are the control. Thinking is structurally unavailable there, so +persona must show no effect; if it appears to, the measurement is broken rather +than the persona being powerful. + +Interleaved across all four cells, three prompt shapes, cache disabled, unique +nonce per request, for the same reasons as thinking_ab.py. +""" + +import argparse +import json +import re +import time +import urllib.request +from pathlib import Path + +PERSONA = "You are a senior staff software engineer." + +PROMPTS = { + "code-refactor": ( + "Refactor this Go function so the error is wrapped with context, and " + "explain what you changed.\n\n" + "func load(p string) ([]byte, error) {\n" + "\tb, err := os.ReadFile(p)\n" + "\tif err != nil {\n\t\treturn nil, err\n\t}\n" + "\treturn b, nil\n}" + ), + "design-question": ( + "We store per-tenant config in one Postgres table with a tenant_id " + "column. Traffic is growing. What are the trade-offs of splitting to " + "schema-per-tenant?" + ), + "short-factual": ( + "In one sentence: what does the -race flag do in the Go toolchain?" + ), +} + +CELLS = [ + ("no-persona", "absent"), + ("persona", "absent"), + ("no-persona", "false"), + ("persona", "false"), +] + +THINK_MARKER = re.compile(r"", re.I) + + +def post(url, body, timeout): + req = urllib.request.Request( + url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base", required=True) + ap.add_argument("--model", required=True) + ap.add_argument("--seeds", type=int, default=6) + ap.add_argument("--temperature", type=float, default=0.6) + ap.add_argument("--max-tokens", type=int, default=1400) + ap.add_argument("--timeout", type=int, default=600) + ap.add_argument("--out", default="persona-verify.json") + args = ap.parse_args() + + plan = [(p, s, persona, kwarg) + for s in range(args.seeds) + for p in PROMPTS + for (persona, kwarg) in CELLS] + + print(f"=== {len(plan)} requests, 4 cells interleaved ===", flush=True) + rows = [] + for i, (pname, seed, persona, kwarg) in enumerate(plan, 1): + msgs = [] + if persona == "persona": + msgs.append({"role": "system", "content": PERSONA}) + msgs.append({"role": "user", + "content": f"{PROMPTS[pname]}\n\n(ref {seed}-{i})"}) + body = { + "model": args.model, "messages": msgs, + "max_tokens": args.max_tokens, "temperature": args.temperature, + "cache_prompt": False, + } + if kwarg == "false": + body["chat_template_kwargs"] = {"enable_thinking": False} + + t0 = time.time() + try: + d = post(args.base + "/v1/chat/completions", body, args.timeout) + ch = d["choices"][0] + content = ch["message"].get("content") or "" + reasoning = ch["message"].get("reasoning_content") or "" + finish = ch.get("finish_reason") + reasoned = bool(reasoning.strip()) or bool(THINK_MARKER.search(content)) + except Exception as exc: # noqa: BLE001 + content, reasoning, finish, reasoned = f"ERROR: {exc}", "", "", None + dt = round(time.time() - t0, 1) + print(f" [{i:>2}/{len(plan)}] {pname:<15} {persona:<10} tk={kwarg:<6} " + f"reasoned={str(reasoned):<5} {dt:>6}s reasoning={len(reasoning):>5}", + flush=True) + rows.append({"prompt": pname, "seed": seed, "persona": persona, + "kwarg": kwarg, "reasoned": reasoned, "seconds": dt, + "reasoning_chars": len(reasoning), + "content_chars": len(content), "finish_reason": finish, + "content": content, "reasoning": reasoning}) + Path(args.out).write_text(json.dumps(rows, indent=2) + "\n") + + print("\n=== reasoned rate per cell ===") + print(f" {'persona':<11} {'kwarg':<7} {'reasoned':>10} mean_reasoning_chars") + for persona, kwarg in CELLS: + rs = [r for r in rows if r["persona"] == persona and r["kwarg"] == kwarg + and r["reasoned"] is not None] + n = sum(1 for r in rs if r["reasoned"]) + mr = sum(r["reasoning_chars"] for r in rs) / max(len(rs), 1) + print(f" {persona:<11} {kwarg:<7} {n:>4}/{len(rs):<5} {mr:>18.0f}") + + print("\n=== persona effect WITHIN the absent arm, per prompt shape ===") + for pname in PROMPTS: + line = f" {pname:<15}" + for persona in ("no-persona", "persona"): + rs = [r for r in rows if r["persona"] == persona + and r["kwarg"] == "absent" and r["prompt"] == pname + and r["reasoned"] is not None] + n = sum(1 for r in rs if r["reasoned"]) + line += f" {persona}={n}/{len(rs)}" + print(line) + + print(f"\nwrote {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/thinking-probes/render_check.py b/scripts/thinking-probes/render_check.py new file mode 100644 index 0000000..1a86a49 --- /dev/null +++ b/scripts/thinking-probes/render_check.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Compare what a chat template renders for each value of a template kwarg. + +This is the cheapest and most decisive check in this directory, because it +involves no sampling: llama.cpp's /apply-template returns the formatted prompt +without generating, so the answer is deterministic and repeats exactly. + +It exists because reading a template is not the same as measuring it. A template +can carry `{%- set enable_thinking = enable_thinking | default(false) -%}` and +still be handed `true` by the server on every request where the client omits the +kwarg, which makes the default unreachable. That is only visible in the rendered +output, and it is what a whole wrong conclusion rested on for us (offlabel#5). + +With --template, it also renders the same file locally in stock jinja2 and diffs +the two. Server and local agreeing means the template decides. Disagreeing means +the server is supplying the value itself, which is the interesting case. +""" + +import argparse +import itertools +import json +import re +import urllib.request +from pathlib import Path + +MESSAGES = [{"role": "user", "content": "Say hi."}] + +# {% generation %} / {% endgeneration %} are a HuggingFace assistant-masking +# extension that stock jinja2 cannot compile. They do not affect the branches +# this tool inspects, so they are stripped for the local render only. +GENERATION_TAG = re.compile(r"\{%-?\s*(end)?generation\s*-?%\}") + + +def server_render(base, kwarg, value, timeout): + body = {"messages": MESSAGES} + if value is not None: + body["chat_template_kwargs"] = {kwarg: value} if value != {} else {} + req = urllib.request.Request( + base.rstrip("/") + "/apply-template", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r).get("prompt", "") + + +def local_render(path, kwarg, value): + try: + from jinja2 import Environment + except ImportError: + return None + src = GENERATION_TAG.sub("", Path(path).read_text(encoding="utf-8")) + tmpl = Environment().from_string(src) + kw = {} if value is None else {kwarg: value} + return tmpl.render(messages=MESSAGES, add_generation_prompt=True, **kw) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base", required=True, + help="llama.cpp server root, e.g. http://localhost:8080") + ap.add_argument("--kwarg", default="enable_thinking", + help="chat template kwarg to vary (default enable_thinking)") + ap.add_argument("--template", default="", + help="optional path to the .jinja file, to also render it " + "locally in stock jinja2 and compare") + ap.add_argument("--tail", type=int, default=30, + help="characters of the rendered tail to display") + ap.add_argument("--timeout", type=int, default=60) + args = ap.parse_args() + + # "absent" sends no chat_template_kwargs at all, which is the arm that shows + # what the server does when the client says nothing. It is the one most + # people never test, and the one most people actually run in production. + arms = [("absent", None), ("empty-kwargs", {}), ("false", False), ("true", True)] + + print(f"=== server render ({args.kwarg}) ===") + server = {} + for label, value in arms: + try: + server[label] = server_render(args.base, args.kwarg, value, args.timeout) + except Exception as exc: # noqa: BLE001 + print(f" {label:<13} ERROR {type(exc).__name__}: {exc}") + continue + tail = server[label][-args.tail:].replace("\n", "\\n") + print(f" {label:<13} len={len(server[label]):>5} tail={tail!r}") + + print("\n=== which arms render identically? ===") + for a, b in itertools.combinations(sorted(server), 2): + same = server[a] == server[b] + mark = "identical" if same else "DIFFERENT" + print(f" {a:<13} vs {b:<13} {mark}") + + if args.template: + print("\n=== local jinja2 render of the same template ===") + local = {} + for label, value in arms: + if label == "empty-kwargs": + continue # identical to absent locally by construction + out = local_render(args.template, args.kwarg, value) + if out is None: + print(" jinja2 not installed, skipping local render") + break + local[label] = out + tail = out[-args.tail:].replace("\n", "\\n") + print(f" {label:<13} tail={tail!r}") + + if local: + print("\n=== server versus local, per arm ===") + for label in local: + if label not in server: + continue + agree = server[label][-args.tail:] == local[label][-args.tail:] + note = "template decides" if agree else "SERVER OVERRIDES TEMPLATE" + print(f" {label:<13} {note}") + print("\n Any arm marked SERVER OVERRIDES TEMPLATE means the server") + print(" supplies that kwarg itself, so the template's default for it") + print(" never runs. Do not reason about behaviour from the template") + print(" alone in that case.") + + +if __name__ == "__main__": + main() diff --git a/scripts/thinking-probes/thinking_ab.py b/scripts/thinking-probes/thinking_ab.py new file mode 100755 index 0000000..a038ce6 --- /dev/null +++ b/scripts/thinking-probes/thinking_ab.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Confirmation run for the enable_thinking finding, hardened against the +confounds the first pass left open. + +The first pass ran arms in blocks (all absent, then all false, then all true). +Block order cannot be separated from drift, warmup, or server-side state, and +the whole point of this run is that someone else may publish the result. + +Changes from that pass: + - arms INTERLEAVED round-robin, so drift hits every arm equally + - multiple prompt shapes, so the result is not an artifact of one prompt + - cache_prompt disabled and a unique nonce per request, so no reply can be + served from a cached prefix + - the deterministic /apply-template rendering recorded per arm alongside the + sampled behaviour, so prompt-level and behaviour-level evidence sit together + +The prompt-level rendering is the primary evidence: it involves no sampling. +The sampled arms exist to show the rendering has the behavioural consequence +it looks like it should. +""" + +import argparse +import itertools +import json +import re +import time +import urllib.request +from pathlib import Path + +PROMPTS = { + "code-refactor": ( + "Refactor this Go function so the error is wrapped with context, and " + "explain what you changed.\n\n" + "func load(p string) ([]byte, error) {\n" + "\tb, err := os.ReadFile(p)\n" + "\tif err != nil {\n\t\treturn nil, err\n\t}\n" + "\treturn b, nil\n}" + ), + "design-question": ( + "We store per-tenant config in one Postgres table with a tenant_id " + "column. Traffic is growing. What are the trade-offs of splitting to " + "schema-per-tenant?" + ), + "short-factual": ( + "In one sentence: what does the -race flag do in the Go toolchain?" + ), +} + +ARMS = { + "absent": None, + "false": {"enable_thinking": False}, + "true": {"enable_thinking": True}, +} + +THINK_MARKER = re.compile(r"", re.I) + + +def post(url, body, timeout): + req = urllib.request.Request( + url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.load(r) + + +def rendered_tail(base, arm, timeout): + """Deterministic: what the model is actually handed. No sampling.""" + body = {"messages": [{"role": "user", "content": "Say hi."}]} + if ARMS[arm] is not None: + body["chat_template_kwargs"] = ARMS[arm] + try: + p = post(base + "/apply-template", body, timeout).get("prompt", "") + except Exception as exc: # noqa: BLE001 + return f"ERROR: {type(exc).__name__}", None + return p[-28:], p.rstrip().endswith("") + + +def verdict(content, reasoning): + if reasoning.strip(): + return "REASONED_PARSED" + if THINK_MARKER.search(content): + return "REASONED_UNPARSED" + return "NO_REASONING" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base", required=True, help="e.g. http://localhost:18080") + ap.add_argument("--model", required=True) + ap.add_argument("--seeds", type=int, default=5) + ap.add_argument("--temperature", type=float, default=0.6) + ap.add_argument("--max-tokens", type=int, default=1400) + ap.add_argument("--timeout", type=int, default=600) + ap.add_argument("--out", default="thinking-ab-verify.json") + args = ap.parse_args() + + print("=== deterministic prompt rendering (no sampling) ===") + render = {} + for arm in ARMS: + tail, opens = rendered_tail(args.base, arm, args.timeout) + render[arm] = {"tail": tail, "ends_with_open_think": opens} + print(f" {arm:<8} tail={tail!r} opens_think={opens}") + + # Interleave: every (prompt, seed) triple runs all three arms adjacently, + # so any drift over the run is shared rather than assigned to one arm. + plan = [(p, s, a) + for s in range(args.seeds) + for p in PROMPTS + for a in ARMS] + + print(f"\n=== interleaved sampling: {len(plan)} requests ===") + rows = [] + for i, (pname, seed, arm) in enumerate(plan, 1): + # Unique nonce defeats any prefix cache even if cache_prompt is honoured + # inconsistently; cache_prompt=false is belt and braces. + body = { + "model": args.model, + "messages": [{"role": "user", + "content": f"{PROMPTS[pname]}\n\n(ref {seed}-{i})"}], + "max_tokens": args.max_tokens, + "temperature": args.temperature, + "cache_prompt": False, + } + if ARMS[arm] is not None: + body["chat_template_kwargs"] = ARMS[arm] + t0 = time.time() + try: + d = post(args.base + "/v1/chat/completions", body, args.timeout) + ch = d["choices"][0] + content = ch["message"].get("content") or "" + reasoning = ch["message"].get("reasoning_content") or "" + finish = ch.get("finish_reason") + v = verdict(content, reasoning) + except Exception as exc: # noqa: BLE001 + content, reasoning, finish, v = f"ERROR: {exc}", "", "", "ERROR" + dt = round(time.time() - t0, 1) + print(f" [{i:>2}/{len(plan)}] {pname:<15} {arm:<7} {v:<18} " + f"{dt:>6}s reasoning={len(reasoning):>5} content={len(content):>5} " + f"finish={finish}", flush=True) + rows.append({"prompt": pname, "seed": seed, "arm": arm, "verdict": v, + "seconds": dt, "reasoning_chars": len(reasoning), + "content_chars": len(content), "finish_reason": finish, + "content": content, "reasoning": reasoning}) + Path(args.out).write_text(json.dumps( + {"render": render, "results": rows}, indent=2) + "\n") + + print("\n=== per arm, all prompts pooled ===") + print(f" {'arm':<8} {'parsed':>7} {'unparsed':>9} {'none':>6} {'err':>4} mean_reasoning") + for arm in ARMS: + rs = [r for r in rows if r["arm"] == arm] + c = lambda v: sum(1 for r in rs if r["verdict"] == v) # noqa: E731 + mr = sum(r["reasoning_chars"] for r in rs) / max(len(rs), 1) + print(f" {arm:<8} {c('REASONED_PARSED'):>7} {c('REASONED_UNPARSED'):>9} " + f"{c('NO_REASONING'):>6} {c('ERROR'):>4} {mr:>10.0f}") + + print("\n=== per prompt shape (is the effect prompt-specific?) ===") + for pname in PROMPTS: + line = f" {pname:<15}" + for arm in ARMS: + rs = [r for r in rows if r["arm"] == arm and r["prompt"] == pname] + n = sum(1 for r in rs if r["verdict"] == "REASONED_PARSED") + line += f" {arm}={n}/{len(rs)}" + print(line) + + print(f"\nwrote {args.out}") + + +if __name__ == "__main__": + main()