feat(eval): add kb effectiveness outcome scoring

CHANGELOG.md

@@ -6,6 +6,17 @@ All notable changes to vouch are documented here. Format follows
 
 ## [Unreleased]
 
+### Added
+- `vouch health effectiveness` and `kb.effectiveness`: a read-only
+  per-artifact outcome signal that correlates context-pack surfacing with
+  session outcomes over a time window. Reports surfaced good/bad counts,
+  lift vs baseline, 95% wilson interval, and conservative verdicts
+  (`useful`/`harmful` only when confidence clears baseline and sample size is
+  sufficient; otherwise `unverified`/`insufficient`).
+- derived `context_surface` cache in `.vouch/state.db`, populated from
+  `build_context_pack` and cleared by `index_db.reset`, to record which
+  artifacts were surfaced to which sessions without changing durable yaml/audit
+  authority.
 ## [1.2.2] — 2026-07-07
 
 ### Packaging

docs/effectiveness.md

@@ -0,0 +1,34 @@
+# Effectiveness schema
+
+`vouch health effectiveness --format json` and `kb.effectiveness` share one
+stable machine-readable schema.
+
+## Top-level fields
+
+- `schema_version` (int): schema revision for compatibility checks.
+- `window` (object): `{spec, since, until, generated_at}`.
+- `min_samples` (int): minimum surfaced samples required for confident verdicts.
+- `sessions` (object): `{classified, good, bad, baseline_rate}`.
+- `artifacts` (array): ranked per-artifact rows.
+
+## Artifact row
+
+- `artifact_kind` (string)
+- `artifact_id` (string)
+- `samples` (int): surfaced sessions with classified outcomes.
+- `surfaced` (object): `{good, bad}` counts for surfaced sessions.
+- `not_surfaced` (object): `{good, bad}` counts for classified sessions where
+  this artifact was not surfaced.
+- `rate` (float): surfaced good rate.
+- `baseline_rate` (float): global good rate across classified sessions.
+- `lift` (float): `rate - baseline_rate`.
+- `wilson_95` (object): `{low, high}` confidence interval on surfaced good rate.
+- `verdict` (string): one of `useful`, `harmful`, `unverified`, `insufficient`.
+- `earned_value` (float): `lift * samples`, used for ranking.
+
+## Verdict rules
+
+- `insufficient`: `samples < min_samples`
+- `useful`: interval lower bound is greater than baseline
+- `harmful`: interval upper bound is lower than baseline
+- `unverified`: otherwise

src/vouch/capabilities.py

@@ -35,6 +35,7 @@
     "kb.search",
     "kb.neighbors",
     "kb.context",
+    "kb.effectiveness",
     "kb.synthesize",
     "kb.read_page",
     "kb.read_claim",

src/vouch/cli.py

@@ -47,6 +47,7 @@
 from . import verify as verify_mod
 from .capabilities import capabilities as build_caps
 from .context import build_context_pack
+from .eval.effectiveness import compute_effectiveness
 from .lifecycle import LifecycleError
 from .logging_config import configure_logging
 from .models import Proposal, ProposalKind, ProposalStatus
@@ -421,6 +422,65 @@ def fsck() -> None:
     sys.exit(0 if report.ok else 1)
 
 
+@cli.group(name="health")
+def health_group() -> None:
+    """Health-oriented diagnostics and evaluation commands."""
+
+
+@health_group.command("effectiveness")
+@click.option(
+    "--window",
+    default="90d",
+    show_default=True,
+    help="Time window (e.g. 90d, 12h, 2w, or ISO date).",
+)
+@click.option(
+    "--min-samples",
+    default=5,
+    show_default=True,
+    type=int,
+    help="Minimum surfaced sessions before a confident verdict is allowed.",
+)
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["text", "json"]),
+    default="text",
+    show_default=True,
+)
+def health_effectiveness(window: str, min_samples: int, output_format: str) -> None:
+    """Measure whether surfaced artifacts correlate with better outcomes."""
+    store = _load_store()
+    with _cli_errors():
+        report = compute_effectiveness(
+            store,
+            window=window,
+            min_samples=min_samples,
+        )
+    if output_format == "json":
+        _emit_json(report)
+        return
+
+    sessions = report["sessions"]
+    base = sessions["baseline_rate"]
+    base_str = "n/a" if base is None else f"{base * 100:.1f}%"
+    click.echo(
+        f"effectiveness window={report['window']['spec']} "
+        f"classified={sessions['classified']} baseline={base_str}"
+    )
+    if not report["artifacts"]:
+        click.echo("no surfaced artifacts with enough classified sessions")
+        return
+    for row in report["artifacts"]:
+        lo = row["wilson_95"]["low"]
+        hi = row["wilson_95"]["high"]
+        click.echo(
+            f"{row['artifact_kind']}/{row['artifact_id']}  "
+            f"verdict={row['verdict']}  samples={row['samples']}  "
+            f"lift={row['lift']:+.3f}  ci95=[{lo:.3f},{hi:.3f}]"
+        )
+
+
 @cli.group(invoke_without_command=True)
 @click.option("--check", "check_only", is_flag=True, help="Only check if a migration is needed.")
 @click.option("--dry-run", is_flag=True, help="Show planned format changes without writing.")

src/vouch/context.py

@@ -250,6 +250,7 @@ def build_context_pack(
     graph_depth: int = 1,
     graph_limit: int = 20,
     graph_rel_types: list[str] | None = None,
+    session_id: str | None = None,
 ) -> ContextPack | dict[str, Any]:
     viewer = viewer_from(
         config_path=store.config_path,
@@ -344,6 +345,20 @@ def build_context_pack(
     )
 
     pack = ContextPack(query=query, items=items, quality=quality, warnings=warnings)
+    if items:
+        try:
+            with index_db.open_db(store.kb_dir) as conn:
+                index_db.index_context_surface(
+                    conn,
+                    session_id=session_id,
+                    query=query,
+                    surfaced_at=pack.generated_at.isoformat(),
+                    items=((item.type, item.id) for item in items),
+                )
+        except sqlite3.Error:
+            # The surfacing log is derived cache only; retrieval must still work
+            # if state.db is unavailable or stale.
+            pass
     result: dict[str, Any] = pack.model_dump()
     result["viewer"] = {
         "project": viewer.project,

src/vouch/eval/__init__.py

@@ -13,10 +13,12 @@
     run_recall,
     score_query,
 )
+from .effectiveness import compute_effectiveness
 
 __all__ = [
     "compare_baseline",
     "load_queries",
     "run_recall",
     "score_query",
+    "compute_effectiveness",
 ]