feat(host): detect CPU steal at callback-deadline kills (detection only)

.changeset/host-steal-detection.md

@@ -0,0 +1,24 @@
+---
+"kit": minor
+---
+
+host: detect CPU steal at callback-deadline kills (detection only)
+
+The per-callback kill switch is wall-clock only, and a true fuel/instruction
+budget is impossible on this stack (wazero v1.12.0 / extism v1.7.1 expose no
+fuel/epoch/gas metering API). So a CPU-steal-throttled VM can blow the deadline
+on a guest running pure, well-behaved guest code, and that kill is booked as a
+fault that can quarantine a healthy game.
+
+This change MEASURES host-stolen CPU at the kill site and records it alongside
+the existing deadline metric — detection only, no behavior change:
+
+- A failure-tolerant, build-tagged steal sampler reads ONLY the /proc/stat
+  aggregate hypervisor-STEAL field (Linux; a no-op ok=false stub elsewhere),
+  sampled at callback BOUNDARIES, never per-instruction. cgroup throttling is
+  deliberately NOT read: steal blames the host (the eventual exonerate case),
+  whereas cgroup throttle blames the guest (a runaway you must not exonerate).
+- A new NON-BREAKING optional `StealMetrics` extension interface is recorded via
+  a type assertion at the wall-clock-kill site, so existing Metrics implementers
+  compile and run unchanged. fault(), quarantine, and End() are untouched; the
+  existing GameCallbackDeadline record is never replaced or suppressed.

host/gameabi/host.go

@@ -792,6 +792,21 @@ func (h *wasmHandler) invoke(r sdk.Room, name string, roster []sdk.Player, extra
 	defer cancel()
 	hostStart := h.cbHostNanos // per-callback host-time delta baseline
 	h.hostIOExpired = false
+	// CPU-steal detection (detection only — see steal.go). Only sample when a
+	// StealMetrics sink is actually wired: the seam is dormant in prod until the
+	// platform implements it, so until then a host pays ZERO /proc/stat reads.
+	// When enabled, sample at callback BOUNDARIES (off the hot per-instruction
+	// path): once before the guest runs, and — only if the callback is killed —
+	// once at the kill site below. If the host-stolen-CPU counter advanced across
+	// that window, the VM was being stolen from while this callback ran. stealOK
+	// additionally guards a host without the signal (non-Linux, or any /proc/stat
+	// read/parse failure) — then no steal metric is recorded.
+	stealSink, stealEnabled := h.game.opts.Metrics.(StealMetrics)
+	var stealPre uint64
+	var stealOK bool
+	if stealEnabled {
+		stealPre, stealOK = stealReader()
+	}
 	cbStart := time.Now()
 	exit, _, err := h.inst.CallWithContext(ctx, name, payload)
 	dur := time.Since(cbStart)
@@ -815,6 +830,18 @@ func (h *wasmHandler) invoke(r sdk.Room, name string, roster []sdk.Player, extra
 			m.GameHostIODeadline(h.game.meta.Slug, name)
 		} else if deadlined {
 			m.GameCallbackDeadline(h.game.meta.Slug, name)
+			// Detection only: alongside (never instead of) the deadline record,
+			// take the SECOND steal sample now — only on a kill, so the happy
+			// path pays nothing. If host-stolen CPU advanced across this killed
+			// callback's window, surface it via the non-breaking optional
+			// StealMetrics seam. The kill still stands and still feeds the fault
+			// path below — this is a correlation signal for the future exonerate
+			// case, not a decision.
+			if stealEnabled && stealOK {
+				if stealPost, ok := stealReader(); ok && stealPost > stealPre {
+					stealSink.GameCallbackStealDeadline(h.game.meta.Slug, name)
+				}
+			}
 		}
 	}
 	if err != nil || exit != 0 {

host/gameabi/steal.go

@@ -0,0 +1,40 @@
+package gameabi
+
+// CPU-steal DETECTION (detection only — no behavior change).
+//
+// Why this exists: the per-callback kill switch is WALL-CLOCK only. A true
+// fuel/instruction budget is impossible on this stack — wazero v1.12.0 and
+// extism v1.7.1 expose no fuel/epoch/gas metering API to bound a guest by work
+// done rather than time elapsed — so a CPU-steal-throttled VM (a noisy neighbor
+// on oversubscribed hardware) can blow the wall-clock deadline on a guest that
+// is running PURE, well-behaved guest code, and that kill is booked as a fault
+// that can quarantine a healthy game. (The slow-shared-Postgres analogue is
+// already carved out via hostIOKill.)
+//
+// The only available move is to MEASURE host-stolen CPU at kill time. This file
+// adds that measurement ALONGSIDE the existing GameCallbackDeadline record. It
+// does NOT change fault(), quarantine, End(), or any kill/condemn decision: no
+// exoneration is wired yet. See steal_linux.go for the single signal (/proc/stat
+// hypervisor steal), its blame direction, and its coarseness caveats.
+
+// stealReader returns the cumulative host-stolen CPU time in jiffies and whether
+// that figure is available. It is an INJECTABLE package var (defaulting to the
+// real per-GOOS reader) so tests can stub steal without a real /proc/stat. The
+// host reads it only at CALLBACK BOUNDARIES (before invoking the guest, and at
+// the kill/return) — never on a hot per-instruction path.
+var stealReader = readStealJiffiesLinux
+
+// StealMetrics is a NON-BREAKING optional extension of the Metrics surface. It
+// is deliberately NOT folded into the Metrics interface: doing so would break
+// every existing Metrics implementer (notably the platform's) on the next kit
+// bump. The kill site type-asserts the configured Metrics to StealMetrics and
+// records ONLY if the implementer opts in, so older implementers compile and
+// run unchanged while the seam stays dormant until the platform implements it.
+type StealMetrics interface {
+	// GameCallbackStealDeadline records one wall-clock callback-deadline kill
+	// during which host-stolen CPU advanced — i.e. the VM was being stolen from
+	// across the killed callback's window. Recorded ALONGSIDE (never instead of)
+	// GameCallbackDeadline. A correlation signal for the future exonerate case,
+	// not a kill/quarantine decision.
+	GameCallbackStealDeadline(slug, callback string)
+}

host/gameabi/steal_linux.go

@@ -0,0 +1,74 @@
+//go:build linux
+
+package gameabi
+
+import (
+	"os"
+	"strconv"
+	"strings"
+)
+
+// procStatPath is the file the steal reader parses. A package var (not a const)
+// so a //go:build linux test can point it at a synthetic fixture without a real
+// /proc — the production default is the kernel's live aggregate CPU accounting.
+var procStatPath = "/proc/stat"
+
+// readStealJiffies returns the cumulative hypervisor-STEAL time, in USER_HZ
+// jiffies (~10ms each), that the kernel has accounted across ALL CPUs since
+// boot — the 8th field of the aggregate "cpu " line in /proc/stat:
+//
+//	cpu  user nice system idle iowait irq softirq steal guest guest_nice
+//	     1    2    3      4    5      6   7       8     ...
+//
+// "Steal" is wall-clock time during which a runnable vCPU was NOT scheduled
+// because the HYPERVISOR ran someone else — i.e. the host took CPU away from
+// us. That is the ONLY signal this reader exposes, by deliberate choice:
+//
+//   - Steal blames the HOST (a noisy-neighbor / oversubscribed VM), so a
+//     deadline kill that coincides with advancing steal is the future
+//     EXONERATE case — a well-behaved guest punished for the platform's
+//     scheduling, not a runaway. (No exoneration is wired yet; this is
+//     detection only.)
+//   - cgroup throttled_usec (CPU quota throttling) blames US: the guest
+//     exceeded its OWN cpu.max quota, which is exactly the runaway you must
+//     NOT exonerate. It is therefore intentionally NOT read and NOT OR'd in.
+//
+// Coarseness caveats, by construction:
+//   - HOST-WIDE: the aggregate "cpu " line counts steal across every vCPU and
+//     every tenant process on this VM, not steal attributable to this room's
+//     goroutine. It can only ever say "the VM was being stolen from around the
+//     time this callback ran", never "this callback's CPU was stolen".
+//   - JIFFY RESOLUTION: USER_HZ is typically 100Hz, so the counter advances in
+//     ~10ms steps — comparable to the default 100ms callback deadline. A short
+//     callback can be wholly stolen yet straddle no jiffy tick (false negative);
+//     a long one can tick over a jiffy boundary on unrelated steal (false
+//     positive). Sampling at callback boundaries (not per-instruction) keeps it
+//     cheap at the cost of this granularity.
+//
+// ok is false on ANY read or parse failure (missing file, short line,
+// non-numeric field): the caller treats "no steal info" as the current
+// behavior, recording no steal metric. A guest cannot influence this file.
+func readStealJiffiesLinux() (steal uint64, ok bool) {
+	b, err := os.ReadFile(procStatPath)
+	if err != nil {
+		return 0, false
+	}
+	// The aggregate line is the first line and starts with "cpu " (note the
+	// trailing space — "cpu0", "cpu1", ... are the per-core lines we skip).
+	for _, line := range strings.Split(string(b), "\n") {
+		if !strings.HasPrefix(line, "cpu ") {
+			continue
+		}
+		fields := strings.Fields(line) // ["cpu", user, nice, system, idle, iowait, irq, softirq, steal, ...]
+		// fields[0] == "cpu"; steal is the 8th value AFTER the label => index 8.
+		if len(fields) <= 8 {
+			return 0, false
+		}
+		v, perr := strconv.ParseUint(fields[8], 10, 64)
+		if perr != nil {
+			return 0, false
+		}
+		return v, true
+	}
+	return 0, false
+}

host/gameabi/steal_linux_test.go

@@ -0,0 +1,100 @@
+//go:build linux
+
+package gameabi
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestReadStealJiffiesLinux parses the STEAL field (the 8th value after the
+// "cpu" label) from a synthetic /proc/stat fed via the injectable procStatPath,
+// and pins the failure-tolerant contract: ANY read/parse problem => ok=false.
+func TestReadStealJiffiesLinux(t *testing.T) {
+	write := func(t *testing.T, content string) string {
+		t.Helper()
+		p := filepath.Join(t.TempDir(), "stat")
+		if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
+			t.Fatalf("write fixture: %v", err)
+		}
+		return p
+	}
+	setPath := func(t *testing.T, p string) {
+		t.Helper()
+		prev := procStatPath
+		procStatPath = p
+		t.Cleanup(func() { procStatPath = prev })
+	}
+
+	for _, tc := range []struct {
+		name      string
+		content   string
+		wantSteal uint64
+		wantOK    bool
+	}{
+		{
+			name: "real-shaped line parses the 8th field",
+			//      user nice system idle    iowait irq softirq STEAL guest guest_nice
+			content:   "cpu  100  20   30     40000   5      6   7       4242  9     10\ncpu0 1 2 3 4 5 6 7 8\nintr 0\n",
+			wantSteal: 4242,
+			wantOK:    true,
+		},
+		{
+			name:      "zero steal (unstolen VM) is a valid reading",
+			content:   "cpu  1 2 3 4 5 6 7 0 9 10\n",
+			wantSteal: 0,
+			wantOK:    true,
+		},
+		{
+			name:    "short line (no steal field) => ok=false",
+			content: "cpu  1 2 3 4\n",
+			wantOK:  false,
+		},
+		{
+			name:    "non-numeric steal field => ok=false",
+			content: "cpu  1 2 3 4 5 6 7 xx 9\n",
+			wantOK:  false,
+		},
+		{
+			name:    "no aggregate cpu line => ok=false",
+			content: "cpu0 1 2 3 4 5 6 7 8\nintr 0\n",
+			wantOK:  false,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			setPath(t, write(t, tc.content))
+			got, ok := readStealJiffiesLinux()
+			if ok != tc.wantOK {
+				t.Fatalf("ok = %v, want %v", ok, tc.wantOK)
+			}
+			if ok && got != tc.wantSteal {
+				t.Fatalf("steal = %d, want %d", got, tc.wantSteal)
+			}
+		})
+	}
+
+	// Missing file => ok=false (failure-tolerant, never panics).
+	setPath(t, filepath.Join(t.TempDir(), "does-not-exist"))
+	if _, ok := readStealJiffiesLinux(); ok {
+		t.Fatal("missing /proc/stat reported ok=true, want false")
+	}
+}
+
+// TestReadStealJiffiesMonotonic asserts the LIVE /proc/stat reader is available
+// and non-decreasing on a real Linux host: steal is a cumulative-since-boot
+// counter, so a later read can never be below an earlier one.
+func TestReadStealJiffiesMonotonic(t *testing.T) {
+	// Default path (real /proc/stat) — procStatPath untouched.
+	a, okA := readStealJiffiesLinux()
+	if !okA {
+		t.Skip("live /proc/stat steal field unavailable on this host")
+	}
+	b, okB := readStealJiffiesLinux()
+	if !okB {
+		t.Fatal("second live read failed after a successful first read")
+	}
+	if b < a {
+		t.Fatalf("steal counter went backwards: %d -> %d (must be monotonic)", a, b)
+	}
+}

host/gameabi/steal_other.go

@@ -0,0 +1,12 @@
+//go:build !linux
+
+package gameabi
+
+// readStealJiffiesLinux is the no-op stub on non-Linux hosts: there is no
+// /proc/stat hypervisor-steal accounting to read, so it always reports no steal
+// info (ok=false) and the caller records no steal metric — exactly the current
+// behavior. Production runs on Linux; this keeps the package compiling and the
+// cross-platform stubbed tests honest on darwin/etc.
+func readStealJiffiesLinux() (steal uint64, ok bool) {
+	return 0, false
+}