From d70a5192c6f9c57287472222a7e0385d9565f106 Mon Sep 17 00:00:00 2001 From: perseus <51974392+tcconnally@users.noreply.github.com> Date: Sat, 27 Jun 2026 23:04:35 -0500 Subject: [PATCH] feat(benchmark): deterministic cost/round-trip attribution + reproducible exhibit (#85, #86) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patent-support: quantify the resolve-before-context savings in dollars + round-trips for the §101 technical-effect argument. #85 — Per-call cost/round-trip metering: - plutus_agent/benchmark_cost.py: CallRecord / ApproachRun / attribute() model a benchmark run as metered model calls per approach (agentic baseline vs resolve-before-context). attribute() emits per-approach round-trips, tokens, and USD cost plus the reduction deltas (round-trips eliminated, % round-trips, $ saved, % cost). Cost uses Plutus's existing frozen PRICE_TABLE. - to_csv() / write_exhibit() export JSON + CSV the Perseus exhibit suite embeds. #86 — Reproducible cost-attribution exhibit: - Deterministic pricing path: cost is a pure function of the frozen price table (pricing.PRICE_TABLE_AS_OF) + token counts — no wall-clock, no network. attribute() and to_csv() are byte-identical for identical inputs; emit_exhibit() confines non-determinism to a single generated_at field that can be pinned. - docs/exhibits/SAMPLE-cost-attribution.{json,csv}: committed reproducible reference (6 round-trips -> 1 = 83.33% eliminated; $0.39 -> $0.12 = 69.23% cost reduction for the canonical six-directive task). - docs/benchmark-cost-attribution.md: integration + reproduction guide. Tests: tests/test_benchmark_cost.py (7 tests; exact cost math, determinism, round-trip counts, exhibit reproducibility). Full suite: 269 passed locally. Closes #85, closes #86. --- docs/benchmark-cost-attribution.md | 81 ++++++++ docs/exhibits/SAMPLE-cost-attribution.csv | 3 + docs/exhibits/SAMPLE-cost-attribution.json | 98 ++++++++++ plutus_agent/benchmark_cost.py | 212 +++++++++++++++++++++ tests/test_benchmark_cost.py | 134 +++++++++++++ 5 files changed, 528 insertions(+) create mode 100644 docs/benchmark-cost-attribution.md create mode 100644 docs/exhibits/SAMPLE-cost-attribution.csv create mode 100644 docs/exhibits/SAMPLE-cost-attribution.json create mode 100644 plutus_agent/benchmark_cost.py create mode 100644 tests/test_benchmark_cost.py diff --git a/docs/benchmark-cost-attribution.md b/docs/benchmark-cost-attribution.md new file mode 100644 index 0000000..10f5511 --- /dev/null +++ b/docs/benchmark-cost-attribution.md @@ -0,0 +1,81 @@ +# Benchmark Cost Attribution (Perseus patent support) + +`plutus_agent/benchmark_cost.py` turns a Perseus benchmark run into a +deterministic, dollar-denominated comparison of two ways to assemble the same +context, for the resolve-before-context patent's §101 technical-effect evidence. + +Supports issues #85 (per-call cost/round-trip metering) and #86 (reproducible +cost-attribution exhibit). + +## What it measures + +| Approach | Round-trips | Idea | +|---|---|---| +| `agentic` | N+1 | The model issues one round-trip per context-gathering operation, re-sending a growing prompt each time, then a final answer call. | +| `resolve_before_context` | 1 | Perseus resolves all directives deterministically *offline*; the model is called once. | + +The comparison reports, per approach: round-trips, input/output tokens, and USD +cost; plus the reduction deltas (round-trips eliminated, % round-trips, $ saved, +% cost). + +## Determinism (issue #86) + +- Cost is computed solely from Plutus's **frozen price table** + (`pricing.PRICE_TABLE`, dated `pricing.PRICE_TABLE_AS_OF`) and the run's token + counts. No wall-clock, no network, no live pricing. +- `attribute(...)` is a pure function — its JSON is byte-identical for identical + inputs. +- `emit_exhibit(...)` adds a single `generated_at` field; pin it + (`generated_at="..."`) to make the whole envelope reproducible for golden + tests. `write_exhibit(...)` derives the filename timestamp from that field, so + envelope and filename always agree. + +## Usage + +```python +from plutus_agent.benchmark_cost import ( + AGENTIC, RESOLVE_BEFORE_CONTEXT, ApproachRun, CallRecord, + attribute, write_exhibit, +) + +agentic = ApproachRun(AGENTIC) +for i in range(5): # one gathering round-trip per directive + agentic.record(CallRecord("anthropic", "claude-opus-4-8", + input_tokens=1000 * (i + 1), output_tokens=120, + label=f"gather-{i}")) +agentic.record(CallRecord("anthropic", "claude-opus-4-8", + input_tokens=6000, output_tokens=400, label="final-answer")) + +resolve = ApproachRun(RESOLVE_BEFORE_CONTEXT).record( + CallRecord("anthropic", "claude-opus-4-8", + input_tokens=6000, output_tokens=400, label="single-shot")) + +attribution = attribute("six-directive-context", agentic, resolve) +json_path, csv_path = write_exhibit(attribution, "docs/exhibits") +``` + +The Perseus exhibit suite calls `attribute(...)` with the token counts its own +`--explain` manifest already measures (agentic round-trip count is a structural +property of the directive manifest; tokens are directly measured), then embeds +the resulting JSON/CSV. + +## Reference exhibit + +`docs/exhibits/SAMPLE-cost-attribution.{json,csv}` is a committed, reproducible +reference for the canonical six-directive task: + +| Approach | Round-trips | Cost (USD) | +|---|---|---| +| agentic | 6 | 0.390000 | +| resolve_before_context | 1 | 0.120000 | +| **reduction** | **5 (83.33%)** | **$0.27 saved (69.23%)** | + +Prices per the frozen table as of `2026-06-26` (`claude-opus-4-8`: $15/M input, +$75/M output). Regenerate with pinned timestamp: + +```python +from plutus_agent.benchmark_cost import emit_exhibit, to_csv +env = emit_exhibit(attribution, generated_at="2026-06-28T00:00:00Z") +``` + +Tests: `tests/test_benchmark_cost.py`. diff --git a/docs/exhibits/SAMPLE-cost-attribution.csv b/docs/exhibits/SAMPLE-cost-attribution.csv new file mode 100644 index 0000000..f556f04 --- /dev/null +++ b/docs/exhibits/SAMPLE-cost-attribution.csv @@ -0,0 +1,3 @@ +task,approach,round_trips,input_tokens,output_tokens,cost_usd,price_table_as_of +six-directive-context,agentic,6,21000,1000,0.390000,2026-06-26 +six-directive-context,resolve_before_context,1,6000,400,0.120000,2026-06-26 diff --git a/docs/exhibits/SAMPLE-cost-attribution.json b/docs/exhibits/SAMPLE-cost-attribution.json new file mode 100644 index 0000000..5b1f64f --- /dev/null +++ b/docs/exhibits/SAMPLE-cost-attribution.json @@ -0,0 +1,98 @@ +{ + "attribution": { + "approaches": { + "agentic": { + "calls": [ + { + "cache_read_tokens": 0, + "input_tokens": 1000, + "label": "gather-0", + "model": "claude-opus-4-8", + "output_tokens": 120, + "provider": "anthropic", + "reasoning_tokens": 0 + }, + { + "cache_read_tokens": 0, + "input_tokens": 2000, + "label": "gather-1", + "model": "claude-opus-4-8", + "output_tokens": 120, + "provider": "anthropic", + "reasoning_tokens": 0 + }, + { + "cache_read_tokens": 0, + "input_tokens": 3000, + "label": "gather-2", + "model": "claude-opus-4-8", + "output_tokens": 120, + "provider": "anthropic", + "reasoning_tokens": 0 + }, + { + "cache_read_tokens": 0, + "input_tokens": 4000, + "label": "gather-3", + "model": "claude-opus-4-8", + "output_tokens": 120, + "provider": "anthropic", + "reasoning_tokens": 0 + }, + { + "cache_read_tokens": 0, + "input_tokens": 5000, + "label": "gather-4", + "model": "claude-opus-4-8", + "output_tokens": 120, + "provider": "anthropic", + "reasoning_tokens": 0 + }, + { + "cache_read_tokens": 0, + "input_tokens": 6000, + "label": "final-answer", + "model": "claude-opus-4-8", + "output_tokens": 400, + "provider": "anthropic", + "reasoning_tokens": 0 + } + ], + "cost_usd": 0.39, + "input_tokens": 21000, + "output_tokens": 1000, + "round_trips": 6 + }, + "resolve_before_context": { + "calls": [ + { + "cache_read_tokens": 0, + "input_tokens": 6000, + "label": "single-shot", + "model": "claude-opus-4-8", + "output_tokens": 400, + "provider": "anthropic", + "reasoning_tokens": 0 + } + ], + "cost_usd": 0.12, + "input_tokens": 6000, + "output_tokens": 400, + "round_trips": 1 + } + }, + "price_table_as_of": "2026-06-26", + "reduction": { + "cost_pct": 69.23, + "cost_usd_saved": 0.27, + "round_trips_eliminated": 5, + "round_trips_pct": 83.33 + }, + "task": "six-directive-context" + }, + "deterministic": true, + "exhibit": "cost-attribution", + "generated_at": "2026-06-28T00:00:00Z", + "price_table_as_of": "2026-06-26", + "supports": "Perseus resolve-before-context patent (\u00a7101 technical effect)" +} diff --git a/plutus_agent/benchmark_cost.py b/plutus_agent/benchmark_cost.py new file mode 100644 index 0000000..1de9147 --- /dev/null +++ b/plutus_agent/benchmark_cost.py @@ -0,0 +1,212 @@ +"""Benchmark cost attribution for the Perseus resolve-before-context patent. + +Issues #85 (per-call cost/round-trip metering) and #86 (reproducible +cost-attribution exhibit). + +This module turns a benchmark run — two approaches to assembling the same +context — into a deterministic, dollar-denominated comparison the Perseus +exhibit suite can embed: + + * **agentic baseline**: the model issues one round-trip per context-gathering + operation (N tool calls => N+ model round-trips). + * **resolve-before-context**: Perseus resolves all directives deterministically + *before* the single model call (1 model round-trip regardless of N). + +The §101 technical-effect argument is strongest in dollars + round-trips, not +just tokens. This module computes both, using Plutus's frozen ``PRICE_TABLE`` +(``pricing.PRICE_TABLE_AS_OF``) so the same inputs always yield the same cost — +a reproducible reduction-to-practice artifact. + +Determinism contract (issue #86): + - Cost is computed solely from the frozen price table + the run's token counts; + no wall-clock, no network, no live pricing. + - The emitted exhibit is byte-identical for identical inputs EXCEPT for an + explicit ``generated_at`` timestamp, which is confined to a single field and + can be pinned via ``generated_at=`` for golden-file tests. +""" +from __future__ import annotations + +import csv +import io +import json +from dataclasses import dataclass, field, asdict +from datetime import datetime, timezone +from typing import Optional + +from .pricing import PRICE_TABLE_AS_OF, estimate_cost + +# Approach identifiers used throughout the exhibit. +AGENTIC = "agentic" +RESOLVE_BEFORE_CONTEXT = "resolve_before_context" + + +@dataclass(frozen=True) +class CallRecord: + """One metered model call within a benchmark run. + + A single context-assembly task comprises one or more model round-trips. The + agentic baseline records one CallRecord per tool-gathering round-trip plus a + final answer call; resolve-before-context records exactly one. + """ + provider: str + model: Optional[str] + input_tokens: int + output_tokens: int + cache_read_tokens: int = 0 + reasoning_tokens: int = 0 + # Free-form label, e.g. "discover-tools", "fetch-file", "final-answer". + label: str = "" + + def cost_usd(self, overrides: Optional[dict] = None) -> float: + return estimate_cost( + self.provider, self.model, + self.input_tokens, self.output_tokens, + self.cache_read_tokens, self.reasoning_tokens, + overrides=overrides, + ) + + +@dataclass +class ApproachRun: + """All metered calls for one approach on one task.""" + approach: str + calls: list[CallRecord] = field(default_factory=list) + + def record(self, call: CallRecord) -> "ApproachRun": + self.calls.append(call) + return self + + @property + def round_trips(self) -> int: + """Number of model round-trips = number of metered model calls.""" + return len(self.calls) + + def total_input_tokens(self) -> int: + return sum(c.input_tokens for c in self.calls) + + def total_output_tokens(self) -> int: + return sum(c.output_tokens for c in self.calls) + + def total_cost_usd(self, overrides: Optional[dict] = None) -> float: + return round(sum(c.cost_usd(overrides) for c in self.calls), 6) + + +def _pct_reduction(baseline: float, improved: float) -> float: + """Percent reduction from baseline to improved (0 when baseline is 0).""" + if baseline <= 0: + return 0.0 + return round((baseline - improved) / baseline * 100.0, 2) + + +def attribute( + task: str, + agentic: ApproachRun, + resolve: ApproachRun, + *, + overrides: Optional[dict] = None, +) -> dict: + """Build the deterministic cost-attribution comparison for one task. + + Returns a plain dict (JSON-serializable) with per-approach totals and the + reduction deltas. Pure function of its inputs + the frozen price table — + no timestamp, no I/O — so it is byte-reproducible. + """ + a_cost = agentic.total_cost_usd(overrides) + r_cost = resolve.total_cost_usd(overrides) + a_rt = agentic.round_trips + r_rt = resolve.round_trips + return { + "task": task, + "price_table_as_of": PRICE_TABLE_AS_OF, + "approaches": { + AGENTIC: { + "round_trips": a_rt, + "input_tokens": agentic.total_input_tokens(), + "output_tokens": agentic.total_output_tokens(), + "cost_usd": a_cost, + "calls": [asdict(c) for c in agentic.calls], + }, + RESOLVE_BEFORE_CONTEXT: { + "round_trips": r_rt, + "input_tokens": resolve.total_input_tokens(), + "output_tokens": resolve.total_output_tokens(), + "cost_usd": r_cost, + "calls": [asdict(c) for c in resolve.calls], + }, + }, + "reduction": { + "round_trips_eliminated": a_rt - r_rt, + "round_trips_pct": _pct_reduction(a_rt, r_rt), + "cost_usd_saved": round(a_cost - r_cost, 6), + "cost_pct": _pct_reduction(a_cost, r_cost), + }, + } + + +def to_csv(attribution: dict) -> str: + """Render a cost-attribution dict as deterministic CSV (one row/approach).""" + buf = io.StringIO() + w = csv.writer(buf, lineterminator="\n") + w.writerow([ + "task", "approach", "round_trips", + "input_tokens", "output_tokens", "cost_usd", "price_table_as_of", + ]) + task = attribution["task"] + as_of = attribution["price_table_as_of"] + for name in (AGENTIC, RESOLVE_BEFORE_CONTEXT): + a = attribution["approaches"][name] + w.writerow([ + task, name, a["round_trips"], + a["input_tokens"], a["output_tokens"], + f"{a['cost_usd']:.6f}", as_of, + ]) + return buf.getvalue() + + +def emit_exhibit( + attribution: dict, + *, + generated_at: Optional[str] = None, +) -> dict: + """Wrap an attribution dict as a self-describing exhibit envelope. + + ``generated_at`` is the ONLY non-deterministic field; pass an explicit value + to make the whole envelope reproducible (golden-file tests pin it). When + omitted it defaults to the current UTC time in ISO-8601. + """ + ts = generated_at or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return { + "exhibit": "cost-attribution", + "supports": "Perseus resolve-before-context patent (§101 technical effect)", + "generated_at": ts, + "price_table_as_of": attribution["price_table_as_of"], + "deterministic": True, + "attribution": attribution, + } + + +def write_exhibit( + attribution: dict, + out_dir: str, + *, + generated_at: Optional[str] = None, +) -> tuple[str, str]: + """Write a timestamped JSON + CSV exhibit pair to ``out_dir``. + + Returns ``(json_path, csv_path)``. The timestamp prefix is derived from the + exhibit's ``generated_at`` so the JSON envelope and the filename agree. + """ + import os + + envelope = emit_exhibit(attribution, generated_at=generated_at) + ts = envelope["generated_at"].replace(":", "").replace("-", "") + os.makedirs(out_dir, exist_ok=True) + base = f"{ts}-cost-attribution" + json_path = os.path.join(out_dir, f"{base}.json") + csv_path = os.path.join(out_dir, f"{base}.csv") + with open(json_path, "w", encoding="utf-8") as f: + json.dump(envelope, f, indent=2, sort_keys=True) + f.write("\n") + with open(csv_path, "w", encoding="utf-8") as f: + f.write(to_csv(attribution)) + return json_path, csv_path diff --git a/tests/test_benchmark_cost.py b/tests/test_benchmark_cost.py new file mode 100644 index 0000000..983b827 --- /dev/null +++ b/tests/test_benchmark_cost.py @@ -0,0 +1,134 @@ +"""Tests for benchmark cost attribution (issues #85, #86). + +Verifies the deterministic cost/round-trip comparison between the agentic +baseline and resolve-before-context, and the reproducibility of the emitted +exhibit. +""" +from __future__ import annotations + +import json + +from plutus_agent.benchmark_cost import ( + AGENTIC, + RESOLVE_BEFORE_CONTEXT, + ApproachRun, + CallRecord, + attribute, + emit_exhibit, + to_csv, + write_exhibit, +) +from plutus_agent.pricing import PRICE_TABLE_AS_OF + + +def _scenario(): + """A 5-directive task: agentic needs 6 round-trips, resolve needs 1. + + Agentic: one discovery/fetch round-trip per directive (5) + a final answer + call (1) = 6 model round-trips, each re-sending the growing context. + Resolve-before-context: directives resolved offline, ONE model call. + """ + model = ("anthropic", "claude-opus-4-8") + agentic = ApproachRun(AGENTIC) + # 5 gathering round-trips, each re-sends a growing prompt (cumulative input). + for i in range(5): + agentic.record(CallRecord( + provider=model[0], model=model[1], + input_tokens=1000 * (i + 1), output_tokens=120, + label=f"gather-{i}", + )) + # Final answer call sees the full assembled context. + agentic.record(CallRecord( + provider=model[0], model=model[1], + input_tokens=6000, output_tokens=400, label="final-answer", + )) + + resolve = ApproachRun(RESOLVE_BEFORE_CONTEXT) + # One call: the resolved context is assembled offline, answered in one shot. + resolve.record(CallRecord( + provider=model[0], model=model[1], + input_tokens=6000, output_tokens=400, label="single-shot", + )) + return agentic, resolve + + +def test_round_trip_counts(): + agentic, resolve = _scenario() + assert agentic.round_trips == 6 + assert resolve.round_trips == 1 + + +def test_attribution_costs_and_reduction(): + agentic, resolve = _scenario() + attr = attribute("six-directive-context", agentic, resolve) + + a = attr["approaches"][AGENTIC] + r = attr["approaches"][RESOLVE_BEFORE_CONTEXT] + + # Resolve uses strictly fewer round-trips and costs strictly less. + assert a["round_trips"] == 6 + assert r["round_trips"] == 1 + assert a["cost_usd"] > r["cost_usd"] > 0 + + # Reduction block is internally consistent. + assert attr["reduction"]["round_trips_eliminated"] == 5 + assert attr["reduction"]["cost_usd_saved"] == round(a["cost_usd"] - r["cost_usd"], 6) + assert 0 < attr["reduction"]["cost_pct"] <= 100 + assert attr["price_table_as_of"] == PRICE_TABLE_AS_OF + + +def test_attribution_is_deterministic(): + """Same inputs => byte-identical attribution JSON (no timestamp inside).""" + a1, r1 = _scenario() + a2, r2 = _scenario() + j1 = json.dumps(attribute("t", a1, r1), sort_keys=True) + j2 = json.dumps(attribute("t", a2, r2), sort_keys=True) + assert j1 == j2 + + +def test_exact_cost_math(): + """Cost matches the frozen price table exactly (opus: $15/M in, $75/M out).""" + run = ApproachRun(RESOLVE_BEFORE_CONTEXT).record( + CallRecord("anthropic", "claude-opus-4-8", + input_tokens=1_000_000, output_tokens=1_000_000) + ) + # 1M input * $15/M + 1M output * $75/M = 90.0 + assert run.total_cost_usd() == 90.0 + + +def test_exhibit_is_reproducible_when_timestamp_pinned(): + agentic, resolve = _scenario() + attr = attribute("six-directive-context", agentic, resolve) + e1 = emit_exhibit(attr, generated_at="2026-06-28T00:00:00Z") + e2 = emit_exhibit(attr, generated_at="2026-06-28T00:00:00Z") + assert json.dumps(e1, sort_keys=True) == json.dumps(e2, sort_keys=True) + assert e1["deterministic"] is True + assert e1["generated_at"] == "2026-06-28T00:00:00Z" + + +def test_csv_is_deterministic_and_two_rows(): + agentic, resolve = _scenario() + attr = attribute("six-directive-context", agentic, resolve) + csv1 = to_csv(attr) + csv2 = to_csv(attr) + assert csv1 == csv2 + lines = csv1.strip().split("\n") + assert len(lines) == 3 # header + 2 approaches + assert lines[0].startswith("task,approach,round_trips") + assert AGENTIC in csv1 and RESOLVE_BEFORE_CONTEXT in csv1 + + +def test_write_exhibit_pair(tmp_path): + agentic, resolve = _scenario() + attr = attribute("six-directive-context", agentic, resolve) + jpath, cpath = write_exhibit( + attr, str(tmp_path), generated_at="2026-06-28T00:00:00Z" + ) + assert jpath.endswith("20260628T000000Z-cost-attribution.json") + assert cpath.endswith("20260628T000000Z-cost-attribution.csv") + # Round-trip the JSON envelope. + envelope = json.loads(open(jpath, encoding="utf-8").read()) + assert envelope["exhibit"] == "cost-attribution" + assert envelope["attribution"]["reduction"]["round_trips_eliminated"] == 5 + # CSV has the two approach rows. + assert "agentic" in open(cpath, encoding="utf-8").read()