fix(metering): durable exactly-once receipt for meter/grace; document the no-budget contract (#92, #99)

docs/adr/0037-metering-command-and-litellm-callback.md

@@ -44,12 +44,19 @@ metered calls outside the ledger and chargeback entirely.
   path. Reservation-backed rows are unaffected (their ledger `model`/`labels` stay null and the
   reservation join still wins); a grace backfill also sets no ledger `model`, so it keeps landing
   in the unattributed bucket exactly as before — only a `meter` row uses the ledger-side fallback.
-- **Idempotency via the provider's response id.** `/v1/meter` uses the same `Idempotency-Key` +
-  fingerprint-checked replay contract as every other command (ADR 0031, section 5.1). The natural
-  key for a metered call is the provider's own response id — stable across a caller's retries of
-  the *same* completed call, distinct across different calls — so the LiteLLM callback derives its
-  idempotency key from the response id, falling back to litellm's per-call id and finally a random
-  key only when neither is available.
+- **Idempotency via the provider's response id, backed by a durable receipt.** `/v1/meter` uses the
+  same `Idempotency-Key` + fingerprint-checked replay contract as every other command (ADR 0031,
+  section 5.1). The natural key for a metered call is the provider's own response id — stable across
+  a caller's retries of the *same* completed call, distinct across different calls — so the LiteLLM
+  callback derives its idempotency key from the response id, falling back to litellm's per-call id
+  and finally a random key only when neither is available. Unlike `reserve` (guarded forever by the
+  reservation `UNIQUE(principal_id, idempotency_key)`) or commit/cancel (guarded by the reservation
+  status machine), a meter or grace backfill applies spend with no reservation, so the reaped
+  `idempotency_key` row would be its *only* dedup — and a retry after that row's TTL would re-run
+  `apply_spend` and double-book the spend, possibly in the wrong period. The claim/replay therefore
+  writes to a dedicated, never-reaped `metered_receipt` table (same shape and semantics as
+  `idempotency_key`, persisting for the life of the record like a reservation row), so the
+  exactly-once guarantee holds beyond any retry window (#92).
 - **The LiteLLM callback reads chargeback labels only from `metadata["tollgate_labels"]`.**
   litellm's call `metadata` is shared with litellm-internal bookkeeping (proxy/router keys like
   `user_api_key_alias` or `model_group`), so forwarding it wholesale into chargeback labels would
@@ -94,6 +101,13 @@ metered calls outside the ledger and chargeback entirely.
 - The metering path never denies, so it carries no availability risk for the calls it observes;
   the trade is that an over-budget metered call is recorded as audited overage rather than caught
   before the fact — by construction, since the call already happened before metering runs.
+- A completed call whose principal/project is governed by *no* budget is refused (403
+  `budget_not_found`, per ADR 0020's default-deny on an empty applicable set) rather than booked —
+  so its already-incurred spend is recorded nowhere. This is intended for V1: metering requires a
+  governing budget to attribute spend against, and "never deny" means "never deny on *headroom*,"
+  not "accept ungoverned spend." Booking such spend against a synthesized/unattributed node for
+  chargeback completeness is a deliberate non-goal here, left to a future ADR if operational data
+  shows ungoverned metering is common (#99).
 - Per-class token *provenance* on the ledger (which class each committed/overage micro-USD amount
   came from) remains deferred; a metered row records total input/output token counts the same way
   a grace backfill does, not a class-by-class breakdown.

migrations/versions/0006_metered_receipt.py

@@ -0,0 +1,46 @@
+"""durable metered_receipt table
+
+Adds ``metered_receipt``, a never-reaped idempotency record for the post-charge commands
+(meter, grace backfill). Those commands apply spend with no reservation, so the reaped
+``idempotency_key`` row was their only dedup and a retry after its TTL double-applied spend,
+possibly in the wrong period (#92). This table shares ``idempotency_key``'s shape and
+claim/replay semantics but is never reaped, so the exactly-once guarantee holds beyond any
+retry window. Additive: no existing table or data is touched.
+
+Revision ID: 0006_metered_receipt
+Revises: 0005_meter_ledger
+Create Date: 2026-07-20
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision = "0006_metered_receipt"
+down_revision = "0005_meter_ledger"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "metered_receipt",
+        sa.Column("principal_id", sa.Text(), nullable=False),
+        sa.Column("key", sa.Text(), nullable=False),
+        sa.Column("command_fingerprint", sa.Text(), nullable=False),
+        sa.Column("status", sa.Text(), nullable=True),
+        sa.Column("response", postgresql.JSONB(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.PrimaryKeyConstraint("principal_id", "key", name=op.f("pk_metered_receipt")),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("metered_receipt")

src/tollgate/adapters/postgres/idempotency_repo.py

@@ -14,7 +14,7 @@
 from datetime import datetime
 from typing import Any, Final
 
-from sqlalchemy import delete, select, tuple_, update
+from sqlalchemy import Table, delete, select, tuple_, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncConnection
 
@@ -29,10 +29,16 @@
 
 
 class PostgresIdempotencyRepository:
-    """Per-principal idempotency claim/replay on one bound connection (§5.1, #71)."""
+    """Per-principal idempotency claim/replay on one bound connection (§5.1, #71).
 
-    def __init__(self, conn: AsyncConnection) -> None:
+    The target table is injectable so the same claim/replay/mismatch mechanism serves both the
+    reaped ``idempotency_key`` table (reserve/commit/cancel, whose durable dedup lives elsewhere)
+    and the never-reaped ``metered_receipt`` table (meter/grace, which have no other backstop, #92).
+    """
+
+    def __init__(self, conn: AsyncConnection, table: Table = idempotency_key) -> None:
         self._conn = conn
+        self._table = table
 
     async def claim(self, principal_id: str, key: str, fingerprint: str) -> IdempotencyClaim:
         """Claim ``(principal_id, key)`` for ``fingerprint`` (§5.1); see the port for outcomes.
@@ -45,10 +51,10 @@ async def claim(self, principal_id: str, key: str, fingerprint: str) -> Idempote
         """
         for _attempt in range(_MAX_CLAIM_ATTEMPTS):
             insert_stmt = (
-                pg_insert(idempotency_key)
+                pg_insert(self._table)
                 .values(principal_id=principal_id, key=key, command_fingerprint=fingerprint)
                 .on_conflict_do_nothing(index_elements=["principal_id", "key"])
-                .returning(idempotency_key.c.key)
+                .returning(self._table.c.key)
             )
             claimed = (await self._conn.execute(insert_stmt)).first()
             if claimed is not None:
@@ -57,11 +63,11 @@ async def claim(self, principal_id: str, key: str, fingerprint: str) -> Idempote
             existing = (
                 await self._conn.execute(
                     select(
-                        idempotency_key.c.command_fingerprint,
-                        idempotency_key.c.response,
+                        self._table.c.command_fingerprint,
+                        self._table.c.response,
                     ).where(
-                        idempotency_key.c.principal_id == principal_id,
-                        idempotency_key.c.key == key,
+                        self._table.c.principal_id == principal_id,
+                        self._table.c.key == key,
                     )
                 )
             ).one_or_none()
@@ -80,10 +86,10 @@ async def store_response(
     ) -> None:
         """Cache a command's response on its key row so a later duplicate replays it."""
         stmt = (
-            update(idempotency_key)
+            update(self._table)
             .where(
-                idempotency_key.c.principal_id == principal_id,
-                idempotency_key.c.key == key,
+                self._table.c.principal_id == principal_id,
+                self._table.c.key == key,
             )
             .values(status=status, response=dict(response))
         )
@@ -103,15 +109,15 @@ async def delete_expired(self, cutoff: datetime, limit: int) -> int:
         concurrent reapers pick disjoint oldest-first batches and make progress without deadlocking.
         """
         picked = (
-            select(idempotency_key.c.principal_id, idempotency_key.c.key)
-            .where(idempotency_key.c.created_at < cutoff)
-            .order_by(idempotency_key.c.created_at)
+            select(self._table.c.principal_id, self._table.c.key)
+            .where(self._table.c.created_at < cutoff)
+            .order_by(self._table.c.created_at)
             .limit(limit)
             .with_for_update(skip_locked=True)
         )
         result = await self._conn.execute(
-            delete(idempotency_key).where(
-                tuple_(idempotency_key.c.principal_id, idempotency_key.c.key).in_(picked)
+            delete(self._table).where(
+                tuple_(self._table.c.principal_id, self._table.c.key).in_(picked)
             )
         )
         return result.rowcount

src/tollgate/adapters/postgres/schema.py

@@ -266,3 +266,23 @@ def _enum_check(column: str, values: tuple[str, ...], name: str) -> CheckConstra
     # tick is a full table scan that outgrows the worker statement timeout at volume (#63).
     Index("ix_idempotency_key_created_at", "created_at"),
 )
+
+#: A durable, never-reaped idempotency record for the post-charge commands (meter, grace backfill).
+#: Unlike reserve (guarded forever by the reservation ``UNIQUE(principal_id, idempotency_key)``) and
+#: commit/cancel (guarded by the reservation status machine), those commands apply spend with no
+#: reservation, so the reaped ``idempotency_key`` row was their ONLY dedup — and a retry after its
+#: TTL re-ran ``apply_spend`` and double-applied the spend, possibly in the wrong period (#92). This
+#: row shares ``idempotency_key``'s shape and claim/replay semantics but is never reaped, so it
+#: persists for the life of the record (like a reservation row) and the exactly-once guarantee holds
+#: beyond any retry window. No ``created_at`` index: nothing range-scans it.
+metered_receipt = Table(
+    "metered_receipt",
+    metadata,
+    Column("principal_id", Text, nullable=False),
+    Column("key", Text, nullable=False),
+    Column("command_fingerprint", Text, nullable=False),
+    Column("status", Text, nullable=True),
+    Column("response", JSONB, nullable=True),
+    Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
+    PrimaryKeyConstraint("principal_id", "key"),
+)

src/tollgate/adapters/postgres/unit_of_work.py

@@ -22,6 +22,7 @@
 from tollgate.adapters.postgres.price_book_repo import PostgresPriceBookRepository
 from tollgate.adapters.postgres.reservations_repo import PostgresReservationRepository
 from tollgate.adapters.postgres.reserve_tx import PostgresReserveTransaction
+from tollgate.adapters.postgres.schema import metered_receipt
 
 
 class _PostgresCommandContext:
@@ -31,6 +32,8 @@ def __init__(self, conn: AsyncConnection) -> None:
         self.prices = PostgresPriceBookRepository(conn)
         self.budgets = PostgresBudgetRepository(conn)
         self.idempotency = PostgresIdempotencyRepository(conn)
+        # Same repo over the never-reaped receipt table: the durable dedup for meter/grace (#92).
+        self.metered_receipt = PostgresIdempotencyRepository(conn, table=metered_receipt)
         self.reservations = PostgresReservationRepository(conn)
         self.ledger = PostgresLedgerRepository(conn)
         self.reserve_tx = PostgresReserveTransaction(conn)

src/tollgate/application/handlers/grace.py

@@ -9,7 +9,10 @@
 remaining (committed up to the remaining, excess as audited overage, ADR 0029's split) and
 appends one ``grace_backfill`` ledger row per node carrying both deltas. An empty applicable
 set is rejected: with no governing budget there is no balance to reconcile against. One §5
-transaction; denials raise and roll back; only a success persists its key (§5.1).
+transaction; denials raise and roll back; only a success persists its receipt (§5.1). Like the
+meter, the backfill applies spend with no reservation, so its dedup lives in the never-reaped
+``metered_receipt`` table — the "backfills exactly once" guarantee holds beyond the
+idempotency-key TTL rather than only within it (#92).
 """
 
 from __future__ import annotations
@@ -92,7 +95,9 @@ async def backfill(
         fingerprint = grace_backfill_fingerprint(auth.principal, command)
         principal_id = auth.credential.principal_id
         async with self._uow.begin() as tx:
-            claim = await tx.idempotency.claim(principal_id, command.idempotency_key, fingerprint)
+            claim = await tx.metered_receipt.claim(
+                principal_id, command.idempotency_key, fingerprint
+            )
             if claim.outcome is ClaimOutcome.REPLAY:
                 response = claim.response
                 if response is None:  # pragma: no cover - a committed command always stored one
@@ -135,7 +140,7 @@ async def backfill(
             await tx.ledger.append(entries)
 
             result = GraceBackfillResult(actual_micro=actual, price_book_version=priced.version)
-            await tx.idempotency.store_response(
+            await tx.metered_receipt.store_response(
                 principal_id,
                 command.idempotency_key,
                 RESPONSE_SUCCEEDED,

src/tollgate/application/handlers/meter.py

@@ -7,7 +7,10 @@
 cost against each node's live remaining (committed up to remaining, excess as audited overage,
 ADR 0029's split) and appends one self-describing ``meter`` ledger row per node carrying the
 provider, model, and chargeback labels. An empty applicable set is rejected. One transaction;
-denials raise and roll back; only a success persists its key (section 5.1).
+denials raise and roll back; only a success persists its receipt (section 5.1). Because a meter
+applies spend with no reservation to guard it, that dedup lives in the never-reaped
+``metered_receipt`` table rather than the TTL'd ``idempotency_key``, so a retry stays exactly-once
+beyond any window instead of double-applying the spend once the key ages out (#92).
 """
 
 from __future__ import annotations
@@ -84,7 +87,9 @@ async def meter(self, auth: AuthContext, command: MeterCommand) -> MeterResult:
         principal_id = auth.credential.principal_id
         ref = "truncated" if command.truncated else None
         async with self._uow.begin() as tx:
-            claim = await tx.idempotency.claim(principal_id, command.idempotency_key, fingerprint)
+            claim = await tx.metered_receipt.claim(
+                principal_id, command.idempotency_key, fingerprint
+            )
             if claim.outcome is ClaimOutcome.REPLAY:
                 response = claim.response
                 if response is None:  # pragma: no cover - a committed command always stored one
@@ -130,7 +135,7 @@ async def meter(self, auth: AuthContext, command: MeterCommand) -> MeterResult:
             await tx.ledger.append(entries)
 
             result = MeterResult(actual_micro=actual, price_book_version=priced.version)
-            await tx.idempotency.store_response(
+            await tx.metered_receipt.store_response(
                 principal_id,
                 command.idempotency_key,
                 RESPONSE_SUCCEEDED,

src/tollgate/application/ports.py

@@ -300,6 +300,16 @@ def prices(self) -> PriceBookRepository: ...
     def budgets(self) -> BudgetRepository: ...
     @property
     def idempotency(self) -> IdempotencyRepository: ...
+    @property
+    def metered_receipt(self) -> IdempotencyRepository:
+        """The durable, never-reaped idempotency store for the post-charge commands (#92).
+
+        Same claim/replay/mismatch interface as :attr:`idempotency`, but backed by a table the
+        reaper never touches, so meter/grace stay exactly-once beyond the idempotency-key TTL —
+        they apply spend with no reservation, so the reaped key was otherwise their only dedup.
+        """
+        ...
+
     @property
     def reservations(self) -> ReservationRepository: ...
     @property

tests/integration/conftest.py

@@ -25,7 +25,7 @@
 _DATA_TABLES = (
     "org, team, user_principal, project, api_credential, price_book, price, "
     "budget, budget_alert, budget_balance, reservation, reservation_line, "
-    "ledger, idempotency_key"
+    "ledger, idempotency_key, metered_receipt"
 )
 
 

tests/integration/test_metering_http.py

@@ -165,3 +165,33 @@ async def test_idempotent_replay_returns_the_same_result_without_double_counting
     )
     assert committed_after_second == committed_after_first
     assert await _scalar(committing_engine, "SELECT count(*) FROM ledger WHERE kind='meter'") == 2
+
+
+async def test_meter_dedup_survives_the_idempotency_key_reaper(
+    client: httpx.AsyncClient, committing_engine: AsyncEngine
+) -> None:
+    # A meter applies spend with no reservation, so its dedup once lived only in the idempotency_key
+    # row the reaper deletes at its TTL — a retry after that re-ran apply_spend and double-applied
+    # the spend (#92). The durable metered_receipt persists past the reaper, so the retry replays.
+    await _seed(committing_engine, user_limit=10_000)
+    first = await client.post("/v1/meter", json=_METER_BODY, headers=_headers("idem-meter"))
+    assert first.status_code == 200
+    committed_after_first = await _scalar(
+        committing_engine, "SELECT committed_micro FROM budget_balance WHERE budget_id = 'b-user'"
+    )
+    # The dedup record lives in metered_receipt, not idempotency_key.
+    assert await _scalar(committing_engine, "SELECT count(*) FROM metered_receipt") == 1
+    assert await _scalar(committing_engine, "SELECT count(*) FROM idempotency_key") == 0
+
+    # Simulate the idempotency-key reaper deleting every key past its TTL.
+    async with committing_engine.begin() as conn:
+        await conn.execute(text("DELETE FROM idempotency_key"))
+
+    second = await client.post("/v1/meter", json=_METER_BODY, headers=_headers("idem-meter"))
+    assert second.status_code == 200
+    assert second.json() == first.json()  # exact replay from the durable receipt
+    committed_after_second = await _scalar(
+        committing_engine, "SELECT committed_micro FROM budget_balance WHERE budget_id = 'b-user'"
+    )
+    assert committed_after_second == committed_after_first  # not double-applied past the TTL
+    assert await _scalar(committing_engine, "SELECT count(*) FROM ledger WHERE kind='meter'") == 2

tests/unit/test_cancel.py

@@ -225,6 +225,7 @@ def __init__(
         self.prices = _FakePrices()
         self.budgets = _FakeBudgets()
         self.idempotency = idempotency
+        self.metered_receipt = self.idempotency
         self.reservations = reservations
         self.ledger = ledger
         self.reserve_tx = _StubReserveTx()

tests/unit/test_commit.py

@@ -256,6 +256,7 @@ def __init__(
         self.prices = prices
         self.budgets = _FakeBudgets()
         self.idempotency = idempotency
+        self.metered_receipt = self.idempotency
         self.reservations = reservations
         self.ledger = ledger
         self.reserve_tx = _StubReserveTx()

tests/unit/test_extend.py

@@ -173,6 +173,7 @@ def __init__(self, reservations: _FakeReservations) -> None:
         self.prices = _StubPrices()
         self.budgets = _StubBudgets()
         self.idempotency = _NoIdempotency()
+        self.metered_receipt = self.idempotency
         self.reservations = reservations
         self.ledger = _StubLedger()
         self.reserve_tx = _StubReserveTx()