From a5c74c90642a80d136b82f7f0bfec41b8e79bac8 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 7 Jul 2026 16:46:34 +0900 Subject: [PATCH] docs: mark keyviz hot key topk implemented --- ...k.md => 2026_05_28_implemented_keyviz_hot_key_topk.md} | 2 +- ..._06_11_proposed_hotspot_split_milestone3_automation.md | 2 +- docs/design/2026_06_23_proposed_scaling_roadmap.md | 2 +- internal/admin/keyviz_hotkeys_fanout.go | 2 +- internal/admin/server.go | 2 +- keyviz/hot_keys.go | 2 +- keyviz/sampler.go | 8 ++++---- keyviz/space_saving.go | 2 +- main.go | 2 +- scripts/rolling-update.env.example | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) rename docs/design/{2026_05_28_proposed_keyviz_hot_key_topk.md => 2026_05_28_implemented_keyviz_hot_key_topk.md} (99%) diff --git a/docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md b/docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md similarity index 99% rename from docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md rename to docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md index c63839589..4df02e062 100644 --- a/docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md +++ b/docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md @@ -1,5 +1,5 @@ --- -status: proposed +status: implemented phase: 2-A++ parent_design: docs/admin_ui_key_visualizer_design.md related_design: docs/design/2026_05_25_implemented_keyviz_subrange_sampling.md diff --git a/docs/design/2026_06_11_proposed_hotspot_split_milestone3_automation.md b/docs/design/2026_06_11_proposed_hotspot_split_milestone3_automation.md index 8a7c5abec..8345ce3a4 100644 --- a/docs/design/2026_06_11_proposed_hotspot_split_milestone3_automation.md +++ b/docs/design/2026_06_11_proposed_hotspot_split_milestone3_automation.md @@ -28,7 +28,7 @@ The parent doc §1 states the central gap, and the code confirms it: - **`RecordAccess` is never called from a real request path.** Its only callers are `distribution/engine_test.go` and `distribution/watcher_test.go` (verified by `grep -rn RecordAccess --include='*.go'`). No code in `kv/`, `adapter/`, or `main.go` invokes it. The whole engine-counter detection path — counters, threshold check, `splitRange` — is dead code. - The `splitRange` it would call is midpoint-only (`midpoint(r.Start, r.End)` appends a `0x00` byte, `distribution/engine.go:541-547`) and mutates the **in-memory** engine routes directly, bypassing the catalog. That violates the repo convention that all catalog mutations go through `SplitRange` so the version bumps and watchers fan out (CLAUDE.md "Route catalog mutations must go through `SplitRange`"). It also contradicts the parent doc §3.1.3 requirement "Choose split keys from observed key distribution (not midpoint-only)." -Meanwhile, a **second**, fully-wired load-observation pipeline already exists: the keyviz sampler (`keyviz/sampler.go`). The coordinator calls `Sampler.Observe` per resolved `(RouteID, key)` on both the write path (`groupMutations → observeMutation`, `kv/sharded_coordinator.go:1844` and `:1795-1800`) and the linearizable/lease read path (`observeRead`, `kv/sharded_coordinator.go:1819-1824`). The sampler keeps per-route read/write op counts and byte counts, windowed at a flush interval (`keyviz/flusher.go:19` `RunFlusher`), with an optional per-route Top-K hot-key sketch (`docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md`) and order-preserving sub-range buckets (`docs/design/2026_05_25_implemented_keyviz_subrange_sampling.md`). It is allocation-free on the hot path and already behind `--keyvizEnabled`. +Meanwhile, a **second**, fully-wired load-observation pipeline already exists: the keyviz sampler (`keyviz/sampler.go`). The coordinator calls `Sampler.Observe` per resolved `(RouteID, key)` on both the write path (`groupMutations → observeMutation`, `kv/sharded_coordinator.go:1844` and `:1795-1800`) and the linearizable/lease read path (`observeRead`, `kv/sharded_coordinator.go:1819-1824`). The sampler keeps per-route read/write op counts and byte counts, windowed at a flush interval (`keyviz/flusher.go:19` `RunFlusher`), with an optional per-route Top-K hot-key sketch (`docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md`) and order-preserving sub-range buckets (`docs/design/2026_05_25_implemented_keyviz_subrange_sampling.md`). It is allocation-free on the hot path and already behind `--keyvizEnabled`. M3's job is to close the gap **without building a third counter pipeline**: drive detection off the keyviz sampler (the already-wired, already-sampled signal), feed candidates into a detector with hysteresis/cooldown, and have a scheduler invoke the existing `SplitRange` RPC. The engine's vestigial `RecordAccess`/`splitRange` path is retired (see §3.4). diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index e13424146..a351e6ee0 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -271,7 +271,7 @@ memory each group's private cache/memtable pins. with proposed extensions for cluster fan-out (`2026_04_27_proposed_keyviz_cluster_fanout.md`), subrange sampling (`2026_05_25_implemented_keyviz_subrange_sampling.md`), - hot-key top-K (`2026_05_28_proposed_keyviz_hot_key_topk.md`), and per-cell + hot-key top-K (`2026_05_28_implemented_keyviz_hot_key_topk.md`), and per-cell conflict (implemented). It is the detection signal M3 reuses. Current adapter-direct Redis/DynamoDB/S3 reads that hit `MVCCStore.GetAt` bypass this coordinator sampler, so read-heavy hotspots remain invisible until read-path diff --git a/internal/admin/keyviz_hotkeys_fanout.go b/internal/admin/keyviz_hotkeys_fanout.go index 23522f500..ac43d7fe4 100644 --- a/internal/admin/keyviz_hotkeys_fanout.go +++ b/internal/admin/keyviz_hotkeys_fanout.go @@ -23,7 +23,7 @@ import ( // per-node status surfaced in the response. // // Merge semantics differ from the matrix fan-out — see -// docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md §6: +// docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md §6: // // - count per key: SUM across peers (responses already carry // scaled-to-true estimates from buildHotKeysResponse). diff --git a/internal/admin/server.go b/internal/admin/server.go index af68d367f..00f672130 100644 --- a/internal/admin/server.go +++ b/internal/admin/server.go @@ -68,7 +68,7 @@ type ServerDeps struct { KeyVizFanout *KeyVizFanout // KeyVizHotKeys backs the per-cell Top-K drill-down handler at - // /admin/api/v1/keyviz/hotkeys (design 2026_05_28_proposed_keyviz_hot_key_topk). + // /admin/api/v1/keyviz/hotkeys (design 2026_05_28_implemented_keyviz_hot_key_topk). // Optional: nil makes the route 503 keyviz_disabled, the same shape // as the matrix route's disabled response. A non-nil source whose // HotKeysOptions() reports enabled=false serves 503 hotkeys_disabled diff --git a/keyviz/hot_keys.go b/keyviz/hot_keys.go index b96fadbf0..e1b45f41e 100644 --- a/keyviz/hot_keys.go +++ b/keyviz/hot_keys.go @@ -7,7 +7,7 @@ import ( "time" ) -// Per-cell hot-key drill-down (design 2026_05_28_proposed_keyviz_hot_key_topk). +// Per-cell hot-key drill-down (design 2026_05_28_implemented_keyviz_hot_key_topk). // Off by default; opt in via MemSamplerOptions.HotKeysEnabled. Disabled-case // overhead is one early-return branch on Observe; enabled-case overhead is a // length check, the splitmix64 `nextSampleRoll() % R == 0` sample gate (see diff --git a/keyviz/sampler.go b/keyviz/sampler.go index 9aa3ce19a..b9f8eb644 100644 --- a/keyviz/sampler.go +++ b/keyviz/sampler.go @@ -92,7 +92,7 @@ const ( ) // Defaults / caps for the per-cell hot-key drill-down knobs (see -// docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md §8). All +// docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md §8). All // off-by-default; HotKeysEnabled=false is the binary's existing // behaviour — no extra hot-path cost, no real key bytes retained. const ( @@ -161,7 +161,7 @@ type MemSamplerOptions struct { KeyBucketsPerRoute int // HotKeysEnabled opts in to per-route Top-K hot-key tracking that // backs the heatmap drill-down (Phase 2-A++; see - // docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md). When + // docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md). When // false the hot path adds one early-return branch and nothing else // — disabled-case behaviour is byte-identical to today. When true // the sampler retains REAL key bytes in memory and exposes them on @@ -236,7 +236,7 @@ type MemSampler struct { virtualIDCounter atomic.Uint64 // hotKeys is the per-route Top-K aggregator (design 2026_05_28 - // _proposed_keyviz_hot_key_topk). nil when HotKeysEnabled is false + // _implemented_keyviz_hot_key_topk). nil when HotKeysEnabled is false // — the Observe hot path then skips the feature with one branch. // Read from the hot path; assigned exactly once at construction. hotKeys *hotKeysAggregator @@ -379,7 +379,7 @@ type routeSlot struct { subBuckets []subCounter // hotKeysSnap is the most-recently-published per-route Top-K - // snapshot (design 2026_05_28_proposed_keyviz_hot_key_topk §4). + // snapshot (design 2026_05_28_implemented_keyviz_hot_key_topk §4). // nil until the hot-keys aggregator's first publish for this route. // Drill-down handlers load it lock-free; the aggregator is the // single writer (Store) and deep-copies snapshot contents so a diff --git a/keyviz/space_saving.go b/keyviz/space_saving.go index e2e18bc2d..66edf2723 100644 --- a/keyviz/space_saving.go +++ b/keyviz/space_saving.go @@ -2,7 +2,7 @@ package keyviz // Space-Saving sketch (Metwally et al.) — Misra–Gries-equivalent // heavy-hitter detector backing the per-cell hot-key drill-down (Phase -// 2-A++; see docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md +// 2-A++; see docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md // §3). Maintains at most `capacity` (= `m`) entries; a key with // frequency f over the observed stream of length N is reported with // error ≤ N/m, and any key with f > N/m is guaranteed in the tracked diff --git a/main.go b/main.go index 742af112d..017220b4c 100644 --- a/main.go +++ b/main.go @@ -196,7 +196,7 @@ var ( keyvizHistoryColumns = flag.Int("keyvizHistoryColumns", keyviz.DefaultHistoryColumns, "Maximum matrix columns retained in the keyviz ring buffer (each column = one Step)") keyvizKeyBucketsPerRoute = flag.Int("keyvizKeyBucketsPerRoute", keyviz.DefaultKeyBucketsPerRoute, "Order-preserving sub-range buckets per individual route for the hot-key heatmap; 1 disables sub-bucketing (route-granular, today's behaviour). Capped at 256; memory is ~K*32 bytes/route, so K_max ~= memBudget/(32*keyvizMaxTrackedRoutes)") - // Hot-key drill-down (Phase 2-A++; design 2026_05_28_proposed_keyviz_hot_key_topk). + // Hot-key drill-down (Phase 2-A++; design 2026_05_28_implemented_keyviz_hot_key_topk). // Off by default — the disabled-case adds one early-return branch // to Observe and retains zero real key bytes. When enabled, the // sampler retains actual hot key bytes in memory and exposes them diff --git a/scripts/rolling-update.env.example b/scripts/rolling-update.env.example index d47805b0e..d32d36cb1 100644 --- a/scripts/rolling-update.env.example +++ b/scripts/rolling-update.env.example @@ -152,7 +152,7 @@ KEYVIZ_ENABLED="false" # KEYVIZ_HOT_KEYS_MAX_KEY_LEN — keys longer than this are skipped # BEFORE the sample gate and increment `skipped_long_keys`. # Default 1024, max 4096. -# See docs/design/2026_05_28_proposed_keyviz_hot_key_topk.md §8 for the +# See docs/design/2026_05_28_implemented_keyviz_hot_key_topk.md §8 for the # full memory / overhead table. KEYVIZ_HOT_KEYS_ENABLED="false" # KEYVIZ_HOT_KEYS_PER_ROUTE="64"