From 00a16a0f6ebe916e043f12ebfe033b1c29443e85 Mon Sep 17 00:00:00 2001
From: bootjp <contact@bootjp.me>
Date: Mon, 13 Jul 2026 14:39:33 +0900
Subject: [PATCH 1/6] Share Pebble block cache across stores

---
 .../2026_06_12_proposed_scaling_roadmap.md    | 21 +++----
 .../2026_06_23_proposed_scaling_roadmap.md    | 43 +++++++-------
 store/lsm_store.go                            | 59 ++++++++++++-------
 store/lsm_store_env_test.go                   | 57 ++++++++++++++----
 4 files changed, 115 insertions(+), 65 deletions(-)

diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md
index e2567cbd8..7ad4c7bfb 100644
--- a/docs/design/2026_06_12_proposed_scaling_roadmap.md
+++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md
@@ -109,10 +109,10 @@ Multi-region blockers:
 
 ### 2.3 Storage tier (Pebble)
 
-`store/lsm_store.go`: per-shard `pebble.Open`, default block cache
-`defaultPebbleCacheBytes = 256 MiB` **per store** (not shared).
-WAL sync via `ELASTICKV_FSM_SYNC_MODE` (default `pebble.Sync` on
-FSM apply; `nosync` opt-in).
+`store/lsm_store.go`: per-shard `pebble.Open`, process-wide shared block cache
+`defaultPebbleCacheBytes = 256 MiB` **per node** (shared by all stores in the
+process). WAL sync via `ELASTICKV_FSM_SYNC_MODE` (default `pebble.Sync` on FSM
+apply; `nosync` opt-in).
 
 `store/mvcc_store.go`: encoded as `UserKey ++ 0x00 ++ inverted_TS`,
 `maxSnapshotVersionCount = 1 M`, `maxSnapshotValueSize = 256 MiB`.
@@ -136,8 +136,9 @@ Storage breakage at 1–10 TB/shard:
   is tens of minutes.
 - Pebble L0CompactionThreshold / LBaseMaxBytes / compaction
   concurrency are defaults; write-heavy shards hit stall thresholds.
-  256 MiB block cache per shard × N shards/node = N × 256 MiB
-  resident memory (shared-cache TODO not landed).
+  Shared block cache M1 has landed, so resident block-cache memory is capped per
+  process rather than N × 256 MiB. Shared memtable / compaction concurrency
+  budgeting and per-group cache fairness remain open.
 - MVCC retention 30 min + per-key 1 M version cap means a key
   written ≥ 555/s for 30 min trips the cap; compactor runs every
   5 min so read tail latency spikes during accumulation.
@@ -374,11 +375,11 @@ control-plane (`*_proposed_*` doc TBD).**
   work.
 - Streamed in parallel across the leader's outgoing transport.
 
-**M2 — Shared block cache + per-shard tuning (`*_proposed_*` doc
+**M2 — Shared block-cache follow-ups + per-shard tuning (`*_proposed_*` doc
 TBD).**
-- Land the existing shared-cache TODO: one `pebble.Cache` per node
-  shared across all shards' stores, sized as a per-node config
-  fraction of available RAM (default 25%).
+- M1 shared block cache has landed: one `pebble.Cache` per process is shared
+  across all shards' stores and sized by `ELASTICKV_PEBBLE_CACHE_MB`.
+- Add shared memtable / compaction concurrency budgets across stores.
 - Surface `L0CompactionThreshold`, `LBaseMaxBytes`,
   `MaxConcurrentCompactions` as per-shard config so a write-heavy
   shard can be tuned without touching the cluster default.
diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md
index fe74eda05..79b504d04 100644
--- a/docs/design/2026_06_23_proposed_scaling_roadmap.md
+++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md
@@ -39,13 +39,11 @@ What bounds a single elastickv deployment today:
    is not wired (see §2(b), §2(e)).
 
 2. **Per-group memory.** Each Raft group owns a private Pebble store, and
-   `NewPebbleStore` → `defaultPebbleOptionsWithCache` allocates a *fresh*
-   block cache per store (`store/lsm_store.go:273-279`, `pebble.NewCache(pebbleCacheBytes)`
-   at `:274`), default 256 MiB (`defaultPebbleCacheBytes` at `:66`). N groups
-   on a node means N × 256 MiB of block cache alone, plus N memtables, N WALs,
-   N compaction budgets. The TODO at `store/lsm_store.go:117-120` records the
-   intended fix (a process-wide shared cache plumbed through `NewPebbleStore`)
-   but it is a comment, not a design.
+   `NewPebbleStore` → `defaultPebbleOptionsWithCache` now borrows one
+   process-wide shared block cache through `processPebbleCacheRef`, default
+   256 MiB (`defaultPebbleCacheBytes`). N groups on a node therefore share one
+   block-cache LRU instead of reserving N × 256 MiB for block cache alone.
+   Per-group memtables, WALs, and compaction budgets are still independent.
 
 3. **Leader concentration.** Leadership of each group is elected
    independently by etcd/raft; nothing spreads leaderships across nodes. One
@@ -136,12 +134,11 @@ memory each group's private cache/memtable pins.
   bounds WAL/snapshot growth to O(manifest), which is the data-volume lever
   for the S3 surface specifically. Still proposed; the legacy `BlobKey`-on-Raft
   path is what runs today.
-- **Shared Pebble cache / resource pools — TODO only, promoted to a design
-  item here.** The `store/lsm_store.go:117-120` TODO is the single biggest
-  per-node memory tax as group count grows: N independent 256 MiB caches with
-  no shared eviction pool. This deserves a real design (see §3, Gap 2), not a
-  comment. It blocks high group counts on a single node, which in turn blocks
-  the "many small ranges" model that split (a) produces.
+- **Shared Pebble cache / resource pools — M1 shipped, follow-ups remain.**
+  The block-cache tax is closed by the process-wide `pebble.Cache` in
+  `store/lsm_store.go`, so group count no longer multiplies block-cache memory.
+  Shared memtable / compaction budgets and per-group fairness still need the
+  follow-up design slices in §3, Gap 2.
 
 ### (b) Write throughput
 
@@ -303,14 +300,18 @@ true multi-node multi-group workloads, tracked as a follow-on rather than a
 bootstrap design blocker.
 
 ### Gap 2 — Shared Pebble cache / resource pools
-**Problem.** Each group's store allocates a private 256 MiB block cache
-(`store/lsm_store.go:273-279`); N groups = N × 256 MiB plus N memtables/WALs,
-with no shared eviction. This caps how many groups (hence how many ranges) one
-node can hold, throttling the "many small ranges" model that split produces.
-**Rough milestones:** (M1) process-wide shared `pebble.Cache` plumbed through
-`NewPebbleStore` (the existing TODO), with per-node sizing. (M2) shared
-memtable / compaction concurrency budget across stores. (M3) per-group
-fairness so one hot group cannot evict everyone else's working set.
+**Status.** M1 is closed: `defaultPebbleOptionsWithCache` now borrows a
+process-wide shared `pebble.Cache` via `processPebbleCacheRef`, and
+`NewPebbleStore` / restore reopen paths hold one explicit store/open reference
+that is released on close or reopen. `ELASTICKV_PEBBLE_CACHE_MB` now sizes the
+node-level shared cache rather than a per-store cache.
+
+**Remaining problem.** Each group's store still owns independent memtables,
+WALs, flush scheduling, and compaction concurrency. That still caps how many
+groups one node can hold, but block-cache memory no longer grows as
+N × 256 MiB. **Remaining milestones:** (M2) shared memtable / compaction
+concurrency budget across stores. (M3) per-group fairness so one hot group
+cannot evict everyone else's working set.
 **Depends-on:** none functionally; pairs naturally with Gap 1 (high group
 counts only matter once multi-node groups exist).
 
diff --git a/store/lsm_store.go b/store/lsm_store.go
index 4eee52e9a..c2f6a1b06 100644
--- a/store/lsm_store.go
+++ b/store/lsm_store.go
@@ -60,17 +60,17 @@ const (
 	// exceed maxSnapshotKeySize once the timestamp suffix is appended.
 	maxPebbleEncodedKeySize = maxSnapshotKeySize + timestampSize
 
-	// defaultPebbleCacheBytes is the default Pebble block-cache capacity per
-	// store. Pebble's built-in EnsureDefaults() supplies only 8 MiB, which
+	// defaultPebbleCacheBytes is the default process-wide Pebble block-cache
+	// capacity. Pebble's built-in EnsureDefaults() supplies only 8 MiB, which
 	// is far too small for our workloads: production observed a block-cache
 	// hit rate of 0.003% (1.8B misses vs 58k hits) because the working set
 	// evicted faster than it filled. 256 MiB is a conservative baseline per
-	// shard; operators can override via ELASTICKV_PEBBLE_CACHE_MB.
+	// node; operators can override via ELASTICKV_PEBBLE_CACHE_MB.
 	defaultPebbleCacheBytes int64 = 256 << 20
 
-	// pebbleCacheMBEnv is the env var operators use to override the per-store
-	// Pebble block-cache capacity. Units are MiB, integer only. Malformed or
-	// out-of-range values fall back to the default.
+	// pebbleCacheMBEnv is the env var operators use to override the
+	// process-wide Pebble block-cache capacity. Units are MiB, integer only.
+	// Malformed or out-of-range values fall back to the default.
 	pebbleCacheMBEnv = "ELASTICKV_PEBBLE_CACHE_MB"
 
 	// pebbleCacheMBMin / pebbleCacheMBMax define the accepted range for the
@@ -124,17 +124,18 @@ const (
 	conflictCheckLargeBatchThreshold     = 1024
 )
 
-// pebbleCacheBytes is the effective per-store Pebble block-cache capacity,
+// pebbleCacheBytes is the effective process-wide Pebble block-cache capacity,
 // resolved once at process start. Exposed as a package variable so tests can
-// swap it via setPebbleCacheBytesForTest; production code treats it as
+// swap it via setSmallPebbleCacheForTest; production code treats it as
 // read-only after init().
-//
-// TODO(perf/pebble): introduce a process-wide shared cache plumbed through
-// NewPebbleStore so all shards on a node share one LRU eviction pool rather
-// than each carrying an independent 256 MiB budget. That requires changing
-// NewPebbleStore's signature and is deferred to a follow-up PR.
 var pebbleCacheBytes = defaultPebbleCacheBytes
 
+var (
+	processPebbleCacheMu    sync.Mutex
+	processPebbleCache      *pebble.Cache
+	processPebbleCacheBytes int64
+)
+
 func init() {
 	pebbleCacheBytes = resolvePebbleCacheBytes(os.Getenv(pebbleCacheMBEnv))
 }
@@ -288,20 +289,18 @@ func WithSSTIngestSnapshots(enabled bool) PebbleStoreOption {
 // defaultPebbleOptionsWithCache returns the standard Pebble options used
 // throughout the store (including restores) to ensure consistent behaviour
 // between a freshly opened and a restored/swapped-in database, along with
-// the owned *pebble.Cache handle.
+// the store-owned *pebble.Cache reference.
 //
 // FormatMajorVersion is pinned to ratchet v1-era DBs above pebble v2's
 // FormatMinSupported (FormatFlushableIngest) before the v2 upgrade lands.
 //
-// The returned options carry a freshly-allocated block cache sized from
-// pebbleCacheBytes. pebble.NewCache hands back a refcounted Cache with
-// ref=1; pebble.Open adds one reference, so the caller MUST Unref the
-// returned cache after the DB is closed (or after pebble.Open fails) to
-// fully release the memory. Callers that only need *pebble.Options should
-// still take the cache handle and defer its Unref to avoid leaking a
-// 256 MiB (default) allocation per call.
+// The returned options carry a process-wide shared block cache sized from
+// pebbleCacheBytes. The process holds one base reference; each call adds one
+// store/open reference and returns it to the caller. The caller MUST Unref the
+// returned cache after the DB is closed (or after pebble.Open fails), matching
+// the lifetime rules used by NewPebbleStore, Restore, and temp restore DBs.
 func defaultPebbleOptionsWithCache() (*pebble.Options, *pebble.Cache) {
-	cache := pebble.NewCache(pebbleCacheBytes)
+	cache := processPebbleCacheRef()
 	opts := &pebble.Options{
 		FS:                 vfs.Default,
 		FormatMajorVersion: pebble.FormatVirtualSSTables,
@@ -313,6 +312,22 @@ func defaultPebbleOptionsWithCache() (*pebble.Options, *pebble.Cache) {
 	return opts, cache
 }
 
+func processPebbleCacheRef() *pebble.Cache {
+	processPebbleCacheMu.Lock()
+	defer processPebbleCacheMu.Unlock()
+
+	if processPebbleCache == nil || processPebbleCacheBytes != pebbleCacheBytes {
+		old := processPebbleCache
+		processPebbleCache = pebble.NewCache(pebbleCacheBytes)
+		processPebbleCacheBytes = pebbleCacheBytes
+		if old != nil {
+			old.Unref()
+		}
+	}
+	processPebbleCache.Ref()
+	return processPebbleCache
+}
+
 // NewPebbleStore creates a new Pebble-backed MVCC store.
 func NewPebbleStore(dir string, opts ...PebbleStoreOption) (MVCCStore, error) {
 	fsmOpts, fsmLabel := resolveFSMApplyWriteOpts(os.Getenv(fsmSyncModeEnv))
diff --git a/store/lsm_store_env_test.go b/store/lsm_store_env_test.go
index addf7d8de..fbe2a6732 100644
--- a/store/lsm_store_env_test.go
+++ b/store/lsm_store_env_test.go
@@ -7,15 +7,15 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// setPebbleCacheBytesForTest swaps the package-level pebbleCacheBytes value
-// for the duration of a single test and restores it during t.Cleanup. The
-// real override happens in init() from ELASTICKV_PEBBLE_CACHE_MB; tests use
-// this helper to exercise specific cache sizes without relying on process
+// setSmallPebbleCacheForTest swaps the package-level pebbleCacheBytes value
+// to 16 MiB for the duration of a single test and restores it during
+// t.Cleanup. The real override happens in init() from
+// ELASTICKV_PEBBLE_CACHE_MB; tests use this helper without relying on process
 // env state.
-func setPebbleCacheBytesForTest(t *testing.T, n int64) {
+func setSmallPebbleCacheForTest(t *testing.T) {
 	t.Helper()
 	prev := pebbleCacheBytes
-	pebbleCacheBytes = n
+	pebbleCacheBytes = 16 << 20
 	t.Cleanup(func() { pebbleCacheBytes = prev })
 }
 
@@ -66,23 +66,23 @@ func TestPebbleCacheEnvOverride(t *testing.T) {
 	})
 }
 
-// TestSetPebbleCacheBytesForTestRestores verifies the helper reinstates the
+// TestSetSmallPebbleCacheForTestRestores verifies the helper reinstates the
 // previous value via t.Cleanup so tests that tweak pebbleCacheBytes do not
 // leak state to later tests in the package.
-func TestSetPebbleCacheBytesForTestRestores(t *testing.T) {
+func TestSetSmallPebbleCacheForTestRestores(t *testing.T) {
 	before := pebbleCacheBytes
 	t.Run("inner", func(t *testing.T) {
-		setPebbleCacheBytesForTest(t, 16<<20)
+		setSmallPebbleCacheForTest(t)
 		require.Equal(t, int64(16)<<20, pebbleCacheBytes)
 	})
 	require.Equal(t, before, pebbleCacheBytes)
 }
 
 // TestDefaultPebbleOptionsCarriesCache sanity-checks that the options
-// constructor wires a cache through at the configured size and that Unref
-// is safe to call after closing the DB (the primary lifecycle path).
+// constructor wires the process-shared cache through at the configured size
+// and that Unref is safe for each borrowed store/open reference.
 func TestDefaultPebbleOptionsCarriesCache(t *testing.T) {
-	setPebbleCacheBytesForTest(t, 16<<20)
+	setSmallPebbleCacheForTest(t)
 	opts, cache := defaultPebbleOptionsWithCache()
 	require.NotNil(t, cache)
 	require.Same(t, cache, opts.Cache)
@@ -90,6 +90,39 @@ func TestDefaultPebbleOptionsCarriesCache(t *testing.T) {
 	cache.Unref()
 }
 
+func TestDefaultPebbleOptionsSharesProcessCache(t *testing.T) {
+	setSmallPebbleCacheForTest(t)
+	opts1, cache1 := defaultPebbleOptionsWithCache()
+	defer cache1.Unref()
+	opts2, cache2 := defaultPebbleOptionsWithCache()
+	defer cache2.Unref()
+
+	require.Same(t, cache1, cache2)
+	require.Same(t, cache1, opts1.Cache)
+	require.Same(t, cache1, opts2.Cache)
+}
+
+func TestNewPebbleStoreSharesProcessCache(t *testing.T) {
+	setSmallPebbleCacheForTest(t)
+
+	s1, err := NewPebbleStore(t.TempDir())
+	require.NoError(t, err)
+	defer s1.Close()
+	ps1, ok := s1.(*pebbleStore)
+	require.True(t, ok)
+
+	s2, err := NewPebbleStore(t.TempDir())
+	require.NoError(t, err)
+	defer s2.Close()
+	ps2, ok := s2.(*pebbleStore)
+	require.True(t, ok)
+
+	require.NotNil(t, ps1.cache)
+	require.Same(t, ps1.cache, ps2.cache)
+	require.Equal(t, int64(16)<<20, ps1.BlockCacheCapacityBytes())
+	require.Equal(t, ps1.BlockCacheCapacityBytes(), ps2.BlockCacheCapacityBytes())
+}
+
 // newPebbleStoreWithFSMApplyWriteOptsForTest constructs a pebbleStore
 // (not the MVCCStore interface) with an explicit *pebble.WriteOptions
 // and sync-mode label for the FSM commit path, bypassing the

From 0500937d290ff8fa50aa4d16ad5fb817c274d25b Mon Sep 17 00:00:00 2001
From: bootjp <contact@bootjp.me>
Date: Mon, 13 Jul 2026 17:21:39 +0900
Subject: [PATCH 2/6] monitoring: emit shared cache metrics once

---
 monitoring/pebble.go      | 102 +++++++++++++++++++++++---------------
 monitoring/pebble_test.go |  77 ++++++++++++++++++++++------
 2 files changed, 126 insertions(+), 53 deletions(-)

diff --git a/monitoring/pebble.go b/monitoring/pebble.go
index 59133099d..91d857523 100644
--- a/monitoring/pebble.go
+++ b/monitoring/pebble.go
@@ -16,12 +16,13 @@ import (
 // dashboard.
 //
 // The point-in-time fields (Sublevels, NumFiles, EstimatedDebt,
-// MemTable.*, NumInProgress, BlockCache.Size) are exposed as
-// Prometheus GAUGES — each poll overwrites the previous value.
-// Monotonic fields (Compact.Count, BlockCache.Hits/Misses) are exposed
-// as COUNTERS; the collector emits only the positive delta against the
-// last snapshot so a store reset (Restore/swap) does not produce
-// negative values.
+// MemTable.*, NumInProgress, BlockCache.Size) are exposed as Prometheus
+// GAUGES — each poll overwrites the previous value. LSM / memtable /
+// compaction signals remain group-scoped; the block cache is process-wide and
+// is emitted once per node because all stores share one pebble.Cache.
+// Monotonic fields (Compact.Count, BlockCache.Hits/Misses) are exposed as
+// COUNTERS; the collector emits only the positive delta against the last
+// snapshot so a store reset (Restore/swap) does not produce negative values.
 //
 // Name convention: elastickv_pebble_* to keep a consistent node_id /
 // node_address label prefix with the rest of the registry.
@@ -121,30 +122,30 @@ func newPebbleMetrics(registerer prometheus.Registerer) *PebbleMetrics {
 		blockCacheSizeBytes: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Name: "elastickv_pebble_block_cache_size_bytes",
-				Help: "Current bytes in use by Pebble's block cache.",
+				Help: "Current bytes in use by the process-wide Pebble block cache.",
 			},
-			[]string{"group"},
+			nil,
 		),
 		blockCacheCapacityBytes: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Name: "elastickv_pebble_block_cache_capacity_bytes",
-				Help: "Configured maximum size of Pebble's block cache in bytes. Paired with elastickv_pebble_block_cache_size_bytes so operators can see usage relative to capacity and with the hit/miss counters so they can reason about whether a low hit rate reflects a cold cache or an undersized one.",
+				Help: "Configured maximum size of the process-wide Pebble block cache in bytes. Paired with elastickv_pebble_block_cache_size_bytes so operators can see usage relative to capacity and with the hit/miss counters so they can reason about whether a low hit rate reflects a cold cache or an undersized one.",
 			},
-			[]string{"group"},
+			nil,
 		),
 		blockCacheHitsTotal: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Name: "elastickv_pebble_block_cache_hits_total",
-				Help: "Cumulative block cache hits reported by Pebble.",
+				Help: "Cumulative process-wide block cache hits reported by Pebble.",
 			},
-			[]string{"group"},
+			nil,
 		),
 		blockCacheMissesTotal: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Name: "elastickv_pebble_block_cache_misses_total",
-				Help: "Cumulative block cache misses reported by Pebble.",
+				Help: "Cumulative process-wide block cache misses reported by Pebble.",
 			},
-			[]string{"group"},
+			nil,
 		),
 		fsmApplySyncMode: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
@@ -238,14 +239,18 @@ type PebbleSource struct {
 type PebbleCollector struct {
 	metrics *PebbleMetrics
 
-	mu       sync.Mutex
-	previous map[uint64]pebbleSnapshot
+	mu                 sync.Mutex
+	previous           map[uint64]pebbleSnapshot
+	previousBlockCache pebbleBlockCacheSnapshot
 }
 
 type pebbleSnapshot struct {
-	compactCount     int64
-	blockCacheHits   int64
-	blockCacheMisses int64
+	compactCount int64
+}
+
+type pebbleBlockCacheSnapshot struct {
+	hits   int64
+	misses int64
 }
 
 func newPebbleCollector(metrics *PebbleMetrics) *PebbleCollector {
@@ -293,6 +298,8 @@ func (c *PebbleCollector) observeOnce(sources []PebbleSource) {
 	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	blockCacheObserved := false
+	blockCacheCapacityObserved := false
 	for _, src := range sources {
 		if src.Source == nil {
 			continue
@@ -302,6 +309,13 @@ func (c *PebbleCollector) observeOnce(sources []PebbleSource) {
 			continue
 		}
 		c.observeSource(src, snap)
+		if !blockCacheObserved {
+			c.observeBlockCache(snap)
+			blockCacheObserved = true
+		}
+		if !blockCacheCapacityObserved {
+			blockCacheCapacityObserved = c.observeBlockCacheCapacity(src)
+		}
 	}
 }
 
@@ -324,35 +338,45 @@ func (c *PebbleCollector) observeSource(src PebbleSource, snap *pebble.Metrics)
 	c.metrics.memtableSizeBytes.WithLabelValues(group).Set(float64(snap.MemTable.Size))
 	c.metrics.memtableZombieCount.WithLabelValues(group).Set(float64(snap.MemTable.ZombieCount))
 
-	// Block cache gauges: current usage (always) + configured capacity
-	// (when the source exposes it). Capacity is static for the lifetime
-	// of a DB in practice, but we re-read each tick so operators observe
-	// the new value immediately after a restart with a different
-	// ELASTICKV_PEBBLE_CACHE_MB.
-	c.metrics.blockCacheSizeBytes.WithLabelValues(group).Set(float64(snap.BlockCache.Size))
-	if capSrc, ok := src.Source.(PebbleCacheCapacitySource); ok {
-		if capBytes := capSrc.BlockCacheCapacityBytes(); capBytes > 0 {
-			c.metrics.blockCacheCapacityBytes.WithLabelValues(group).Set(float64(capBytes))
-		}
-	}
-
 	// Monotonic counters: emit only the positive delta. A smaller value
 	// means the source was reset (store reopened); rebase silently
 	// without emitting negative.
 	prev := c.previous[src.GroupID]
 	curr := pebbleSnapshot{
-		compactCount:     snap.Compact.Count,
-		blockCacheHits:   snap.BlockCache.Hits,
-		blockCacheMisses: snap.BlockCache.Misses,
+		compactCount: snap.Compact.Count,
 	}
 	if curr.compactCount > prev.compactCount {
 		c.metrics.compactCountTotal.WithLabelValues(group).Add(float64(curr.compactCount - prev.compactCount))
 	}
-	if curr.blockCacheHits > prev.blockCacheHits {
-		c.metrics.blockCacheHitsTotal.WithLabelValues(group).Add(float64(curr.blockCacheHits - prev.blockCacheHits))
+	c.previous[src.GroupID] = curr
+}
+
+func (c *PebbleCollector) observeBlockCache(snap *pebble.Metrics) {
+	c.metrics.blockCacheSizeBytes.WithLabelValues().Set(float64(snap.BlockCache.Size))
+
+	prev := c.previousBlockCache
+	curr := pebbleBlockCacheSnapshot{
+		hits:   snap.BlockCache.Hits,
+		misses: snap.BlockCache.Misses,
 	}
-	if curr.blockCacheMisses > prev.blockCacheMisses {
-		c.metrics.blockCacheMissesTotal.WithLabelValues(group).Add(float64(curr.blockCacheMisses - prev.blockCacheMisses))
+	if curr.hits > prev.hits {
+		c.metrics.blockCacheHitsTotal.WithLabelValues().Add(float64(curr.hits - prev.hits))
 	}
-	c.previous[src.GroupID] = curr
+	if curr.misses > prev.misses {
+		c.metrics.blockCacheMissesTotal.WithLabelValues().Add(float64(curr.misses - prev.misses))
+	}
+	c.previousBlockCache = curr
+}
+
+func (c *PebbleCollector) observeBlockCacheCapacity(src PebbleSource) bool {
+	capSrc, ok := src.Source.(PebbleCacheCapacitySource)
+	if !ok {
+		return false
+	}
+	capBytes := capSrc.BlockCacheCapacityBytes()
+	if capBytes <= 0 {
+		return false
+	}
+	c.metrics.blockCacheCapacityBytes.WithLabelValues().Set(float64(capBytes))
+	return true
 }
diff --git a/monitoring/pebble_test.go b/monitoring/pebble_test.go
index dc9cb270f..d7e13097a 100644
--- a/monitoring/pebble_test.go
+++ b/monitoring/pebble_test.go
@@ -97,16 +97,16 @@ func TestPebbleCollectorMirrorsGaugesAndCounters(t *testing.T) {
 	err := testutil.GatherAndCompare(
 		registry.Gatherer(),
 		strings.NewReader(`
-# HELP elastickv_pebble_block_cache_hits_total Cumulative block cache hits reported by Pebble.
+# HELP elastickv_pebble_block_cache_hits_total Cumulative process-wide block cache hits reported by Pebble.
 # TYPE elastickv_pebble_block_cache_hits_total counter
-elastickv_pebble_block_cache_hits_total{group="1",node_address="10.0.0.1:50051",node_id="n1"} 150
-# HELP elastickv_pebble_block_cache_misses_total Cumulative block cache misses reported by Pebble.
+elastickv_pebble_block_cache_hits_total{node_address="10.0.0.1:50051",node_id="n1"} 150
+# HELP elastickv_pebble_block_cache_misses_total Cumulative process-wide block cache misses reported by Pebble.
 # TYPE elastickv_pebble_block_cache_misses_total counter
-elastickv_pebble_block_cache_misses_total{group="1",node_address="10.0.0.1:50051",node_id="n1"} 25
+elastickv_pebble_block_cache_misses_total{node_address="10.0.0.1:50051",node_id="n1"} 25
 
-# HELP elastickv_pebble_block_cache_size_bytes Current bytes in use by Pebble's block cache.
+# HELP elastickv_pebble_block_cache_size_bytes Current bytes in use by the process-wide Pebble block cache.
 # TYPE elastickv_pebble_block_cache_size_bytes gauge
-elastickv_pebble_block_cache_size_bytes{group="1",node_address="10.0.0.1:50051",node_id="n1"} 16384
+elastickv_pebble_block_cache_size_bytes{node_address="10.0.0.1:50051",node_id="n1"} 16384
 # HELP elastickv_pebble_compact_count_total Cumulative number of compactions completed by Pebble since the process started.
 # TYPE elastickv_pebble_compact_count_total counter
 elastickv_pebble_compact_count_total{group="1",node_address="10.0.0.1:50051",node_id="n1"} 15
@@ -171,9 +171,9 @@ func TestPebbleCollectorHandlesSourceReset(t *testing.T) {
 	err := testutil.GatherAndCompare(
 		registry.Gatherer(),
 		strings.NewReader(`
-# HELP elastickv_pebble_block_cache_hits_total Cumulative block cache hits reported by Pebble.
+# HELP elastickv_pebble_block_cache_hits_total Cumulative process-wide block cache hits reported by Pebble.
 # TYPE elastickv_pebble_block_cache_hits_total counter
-elastickv_pebble_block_cache_hits_total{group="7",node_address="10.0.0.1:50051",node_id="n1"} 110
+elastickv_pebble_block_cache_hits_total{node_address="10.0.0.1:50051",node_id="n1"} 110
 # HELP elastickv_pebble_compact_count_total Cumulative number of compactions completed by Pebble since the process started.
 # TYPE elastickv_pebble_compact_count_total counter
 elastickv_pebble_compact_count_total{group="7",node_address="10.0.0.1:50051",node_id="n1"} 12
@@ -225,12 +225,12 @@ func TestPebbleCollectorEmitsBlockCacheCapacity(t *testing.T) {
 	err := testutil.GatherAndCompare(
 		registry.Gatherer(),
 		strings.NewReader(`
-# HELP elastickv_pebble_block_cache_capacity_bytes Configured maximum size of Pebble's block cache in bytes. Paired with elastickv_pebble_block_cache_size_bytes so operators can see usage relative to capacity and with the hit/miss counters so they can reason about whether a low hit rate reflects a cold cache or an undersized one.
+# HELP elastickv_pebble_block_cache_capacity_bytes Configured maximum size of the process-wide Pebble block cache in bytes. Paired with elastickv_pebble_block_cache_size_bytes so operators can see usage relative to capacity and with the hit/miss counters so they can reason about whether a low hit rate reflects a cold cache or an undersized one.
 # TYPE elastickv_pebble_block_cache_capacity_bytes gauge
-elastickv_pebble_block_cache_capacity_bytes{group="3",node_address="10.0.0.1:50051",node_id="n1"} 2.68435456e+08
-# HELP elastickv_pebble_block_cache_size_bytes Current bytes in use by Pebble's block cache.
+elastickv_pebble_block_cache_capacity_bytes{node_address="10.0.0.1:50051",node_id="n1"} 2.68435456e+08
+# HELP elastickv_pebble_block_cache_size_bytes Current bytes in use by the process-wide Pebble block cache.
 # TYPE elastickv_pebble_block_cache_size_bytes gauge
-elastickv_pebble_block_cache_size_bytes{group="3",node_address="10.0.0.1:50051",node_id="n1"} 4096
+elastickv_pebble_block_cache_size_bytes{node_address="10.0.0.1:50051",node_id="n1"} 4096
 `),
 		"elastickv_pebble_block_cache_capacity_bytes",
 		"elastickv_pebble_block_cache_size_bytes",
@@ -238,11 +238,60 @@ elastickv_pebble_block_cache_size_bytes{group="3",node_address="10.0.0.1:50051",
 	require.NoError(t, err)
 }
 
+func TestPebbleCollectorEmitsSharedBlockCacheOnce(t *testing.T) {
+	registry := NewRegistry("n1", "10.0.0.1:50051")
+	collector := registry.PebbleCollector()
+	require.NotNil(t, collector)
+
+	src1 := &fakePebbleCapacitySource{capacity: 256 << 20}
+	src2 := &fakePebbleCapacitySource{capacity: 256 << 20}
+	src1.set(newFakeMetrics(
+		1, 2, 0, 0, 7,
+		1, 1024, 0,
+		8192, 100, 20,
+	))
+	src2.set(newFakeMetrics(
+		3, 4, 0, 0, 11,
+		2, 2048, 0,
+		8192, 100, 20,
+	))
+
+	collector.ObserveOnce([]PebbleSource{
+		{GroupID: 1, GroupIDStr: "1", Source: src1},
+		{GroupID: 2, GroupIDStr: "2", Source: src2},
+	})
+
+	err := testutil.GatherAndCompare(
+		registry.Gatherer(),
+		strings.NewReader(`
+# HELP elastickv_pebble_block_cache_capacity_bytes Configured maximum size of the process-wide Pebble block cache in bytes. Paired with elastickv_pebble_block_cache_size_bytes so operators can see usage relative to capacity and with the hit/miss counters so they can reason about whether a low hit rate reflects a cold cache or an undersized one.
+# TYPE elastickv_pebble_block_cache_capacity_bytes gauge
+elastickv_pebble_block_cache_capacity_bytes{node_address="10.0.0.1:50051",node_id="n1"} 2.68435456e+08
+# HELP elastickv_pebble_block_cache_hits_total Cumulative process-wide block cache hits reported by Pebble.
+# TYPE elastickv_pebble_block_cache_hits_total counter
+elastickv_pebble_block_cache_hits_total{node_address="10.0.0.1:50051",node_id="n1"} 100
+# HELP elastickv_pebble_block_cache_misses_total Cumulative process-wide block cache misses reported by Pebble.
+# TYPE elastickv_pebble_block_cache_misses_total counter
+elastickv_pebble_block_cache_misses_total{node_address="10.0.0.1:50051",node_id="n1"} 20
+# HELP elastickv_pebble_block_cache_size_bytes Current bytes in use by the process-wide Pebble block cache.
+# TYPE elastickv_pebble_block_cache_size_bytes gauge
+elastickv_pebble_block_cache_size_bytes{node_address="10.0.0.1:50051",node_id="n1"} 8192
+`),
+		"elastickv_pebble_block_cache_capacity_bytes",
+		"elastickv_pebble_block_cache_size_bytes",
+		"elastickv_pebble_block_cache_hits_total",
+		"elastickv_pebble_block_cache_misses_total",
+	)
+	require.NoError(t, err)
+	require.Equal(t, 1, testutil.CollectAndCount(registry.pebble.blockCacheSizeBytes))
+	require.Equal(t, 1, testutil.CollectAndCount(registry.pebble.blockCacheHitsTotal))
+}
+
 func TestPebbleCollectorSkipsCapacityWhenUnsupported(t *testing.T) {
 	// A PebbleMetricsSource that does NOT additionally implement
 	// PebbleCacheCapacitySource must not cause the capacity gauge to be
-	// populated for its group. This preserves backward compatibility
-	// with sources that pre-date the capacity interface.
+	// populated. This preserves backward compatibility with sources that
+	// pre-date the capacity interface.
 	registry := NewRegistry("n1", "10.0.0.1:50051")
 	collector := registry.PebbleCollector()
 	require.NotNil(t, collector)

From 604354d7a091cba21526aa0b211b4d40e8a60983 Mon Sep 17 00:00:00 2001
From: bootjp <contact@bootjp.me>
Date: Sat, 18 Jul 2026 20:01:24 +0900
Subject: [PATCH 3/6] store: size shared cache from node memory

---
 .../2026_06_12_proposed_scaling_roadmap.md    |  10 +-
 .../2026_06_23_proposed_scaling_roadmap.md    |  11 +-
 store/lsm_store.go                            | 100 +++++++----
 store/lsm_store_env_test.go                   | 165 +++++++++++++++---
 store/pebble_memory_budget.go                 |  67 +++++++
 store/pebble_memory_darwin.go                 |  17 ++
 store/pebble_memory_linux.go                  |  47 +++++
 store/pebble_memory_other.go                  |   7 +
 8 files changed, 360 insertions(+), 64 deletions(-)
 create mode 100644 store/pebble_memory_budget.go
 create mode 100644 store/pebble_memory_darwin.go
 create mode 100644 store/pebble_memory_linux.go
 create mode 100644 store/pebble_memory_other.go

diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md
index 7ad4c7bfb..a32acbbca 100644
--- a/docs/design/2026_06_12_proposed_scaling_roadmap.md
+++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md
@@ -110,9 +110,10 @@ Multi-region blockers:
 ### 2.3 Storage tier (Pebble)
 
 `store/lsm_store.go`: per-shard `pebble.Open`, process-wide shared block cache
-`defaultPebbleCacheBytes = 256 MiB` **per node** (shared by all stores in the
-process). WAL sync via `ELASTICKV_FSM_SYNC_MODE` (default `pebble.Sync` on FSM
-apply; `nosync` opt-in).
+sized to 25% of the node's effective memory budget by default and shared by all
+stores in the process. `ELASTICKV_PEBBLE_CACHE_PERCENT` changes that fraction;
+`ELASTICKV_PEBBLE_CACHE_MB` is the absolute-capacity override. WAL sync via
+`ELASTICKV_FSM_SYNC_MODE` (default `pebble.Sync` on FSM apply; `nosync` opt-in).
 
 `store/mvcc_store.go`: encoded as `UserKey ++ 0x00 ++ inverted_TS`,
 `maxSnapshotVersionCount = 1 M`, `maxSnapshotValueSize = 256 MiB`.
@@ -378,7 +379,8 @@ control-plane (`*_proposed_*` doc TBD).**
 **M2 — Shared block-cache follow-ups + per-shard tuning (`*_proposed_*` doc
 TBD).**
 - M1 shared block cache has landed: one `pebble.Cache` per process is shared
-  across all shards' stores and sized by `ELASTICKV_PEBBLE_CACHE_MB`.
+  across all shards' stores, defaults to 25% of the node's effective memory
+  budget, and supports percentage or absolute-MiB operator overrides.
 - Add shared memtable / compaction concurrency budgets across stores.
 - Surface `L0CompactionThreshold`, `LBaseMaxBytes`,
   `MaxConcurrentCompactions` as per-shard config so a write-heavy
diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md
index 79b504d04..e53d01a19 100644
--- a/docs/design/2026_06_23_proposed_scaling_roadmap.md
+++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md
@@ -40,9 +40,10 @@ What bounds a single elastickv deployment today:
 
 2. **Per-group memory.** Each Raft group owns a private Pebble store, and
    `NewPebbleStore` → `defaultPebbleOptionsWithCache` now borrows one
-   process-wide shared block cache through `processPebbleCacheRef`, default
-   256 MiB (`defaultPebbleCacheBytes`). N groups on a node therefore share one
-   block-cache LRU instead of reserving N × 256 MiB for block cache alone.
+   process-wide shared block cache through `processPebbleCacheRef`, defaulting
+   to 25% of the smallest discovered GOMEMLIMIT, cgroup limit, or physical RAM
+   size. N groups on a node therefore share one block-cache LRU instead of
+   reserving an independent fixed-size cache per group.
    Per-group memtables, WALs, and compaction budgets are still independent.
 
 3. **Leader concentration.** Leadership of each group is elected
@@ -304,7 +305,9 @@ bootstrap design blocker.
 process-wide shared `pebble.Cache` via `processPebbleCacheRef`, and
 `NewPebbleStore` / restore reopen paths hold one explicit store/open reference
 that is released on close or reopen. `ELASTICKV_PEBBLE_CACHE_MB` now sizes the
-node-level shared cache rather than a per-store cache.
+node-level shared cache rather than a per-store cache; when it is unset,
+`ELASTICKV_PEBBLE_CACHE_PERCENT` selects the fraction of the node's effective
+memory budget (25% by default).
 
 **Remaining problem.** Each group's store still owns independent memtables,
 WALs, flush scheduling, and compaction concurrency. That still caps how many
diff --git a/store/lsm_store.go b/store/lsm_store.go
index c2f6a1b06..1e2b58a24 100644
--- a/store/lsm_store.go
+++ b/store/lsm_store.go
@@ -60,19 +60,31 @@ const (
 	// exceed maxSnapshotKeySize once the timestamp suffix is appended.
 	maxPebbleEncodedKeySize = maxSnapshotKeySize + timestampSize
 
-	// defaultPebbleCacheBytes is the default process-wide Pebble block-cache
-	// capacity. Pebble's built-in EnsureDefaults() supplies only 8 MiB, which
-	// is far too small for our workloads: production observed a block-cache
-	// hit rate of 0.003% (1.8B misses vs 58k hits) because the working set
-	// evicted faster than it filled. 256 MiB is a conservative baseline per
-	// node; operators can override via ELASTICKV_PEBBLE_CACHE_MB.
+	// defaultPebbleCacheBytes is the fallback process-wide Pebble block-cache
+	// capacity when the node's effective memory budget cannot be discovered.
+	// Pebble's built-in EnsureDefaults() supplies only 8 MiB, which is far too
+	// small for our workloads: production observed a block-cache hit rate of
+	// 0.003% (1.8B misses vs 58k hits) because the working set evicted faster
+	// than it filled. Operators can set ELASTICKV_PEBBLE_CACHE_MB to override
+	// the percentage-based default with an absolute capacity.
 	defaultPebbleCacheBytes int64 = 256 << 20
 
 	// pebbleCacheMBEnv is the env var operators use to override the
 	// process-wide Pebble block-cache capacity. Units are MiB, integer only.
-	// Malformed or out-of-range values fall back to the default.
+	// Malformed or out-of-range values fall through to percentage sizing.
 	pebbleCacheMBEnv = "ELASTICKV_PEBBLE_CACHE_MB"
 
+	// pebbleCachePercentEnv controls the percentage (1..90) of the node's
+	// effective memory budget reserved for the shared cache when no absolute
+	// MiB override is set. The effective budget is the smallest discovered
+	// GOMEMLIMIT, cgroup limit, or physical-memory size.
+	pebbleCachePercentEnv = "ELASTICKV_PEBBLE_CACHE_PERCENT"
+
+	defaultPebbleCachePercent = 25
+	pebbleCachePercentMin     = 1
+	pebbleCachePercentMax     = 90
+	percentScale              = 100
+
 	// pebbleCacheMBMin / pebbleCacheMBMax define the accepted range for the
 	// env override. Values below the 8 MiB floor or above the 64 GiB ceiling
 	// are rejected and fall back to the default; the ceiling guards against
@@ -137,7 +149,11 @@ var (
 )
 
 func init() {
-	pebbleCacheBytes = resolvePebbleCacheBytes(os.Getenv(pebbleCacheMBEnv))
+	pebbleCacheBytes = resolvePebbleCacheBytes(
+		os.Getenv(pebbleCacheMBEnv),
+		os.Getenv(pebbleCachePercentEnv),
+		effectiveMemoryBudgetBytes(),
+	)
 }
 
 // resolveFSMApplyWriteOpts parses an ELASTICKV_FSM_SYNC_MODE value and
@@ -157,25 +173,44 @@ func resolveFSMApplyWriteOpts(envVal string) (*pebble.WriteOptions, string) {
 	}
 }
 
-// resolvePebbleCacheBytes parses an ELASTICKV_PEBBLE_CACHE_MB value and
-// returns the resolved cache size in bytes. Empty, malformed, or
-// out-of-range values are rejected and fall back to the default rather
-// than being clamped to the nearest bound. "0" is below the 8 MiB floor
-// and therefore also falls back to the default.
-func resolvePebbleCacheBytes(envVal string) int64 {
-	if envVal == "" {
-		return defaultPebbleCacheBytes
+// resolvePebbleCacheBytes resolves the process-wide cache capacity. A valid
+// absolute MiB override wins. Otherwise the configured percentage (25% by
+// default) is applied to the node's effective memory budget. Malformed or
+// out-of-range settings fall back to the percentage default, and an unknown
+// memory budget falls back to defaultPebbleCacheBytes.
+func resolvePebbleCacheBytes(mbEnvVal, percentEnvVal string, memoryBudgetBytes int64) int64 {
+	if absoluteBytes := resolvePebbleCacheAbsoluteBytes(mbEnvVal); absoluteBytes > 0 {
+		return absoluteBytes
 	}
-	mb, err := strconv.Atoi(envVal)
-	if err != nil {
+
+	if memoryBudgetBytes <= 0 {
 		return defaultPebbleCacheBytes
 	}
-	if mb < pebbleCacheMBMin || mb > pebbleCacheMBMax {
-		return defaultPebbleCacheBytes
+
+	cacheBytes := memoryBudgetBytes / percentScale * int64(resolvePebbleCachePercent(percentEnvVal))
+	minimumBytes := int64(pebbleCacheMBMin) << mebibyteShift
+	if cacheBytes < minimumBytes {
+		return minimumBytes
+	}
+	return cacheBytes
+}
+
+func resolvePebbleCacheAbsoluteBytes(envVal string) int64 {
+	mb, err := strconv.Atoi(envVal)
+	if envVal == "" || err != nil || mb < pebbleCacheMBMin || mb > pebbleCacheMBMax {
+		return 0
 	}
 	return int64(mb) << mebibyteShift
 }
 
+func resolvePebbleCachePercent(envVal string) int {
+	configured, err := strconv.Atoi(envVal)
+	if envVal == "" || err != nil || configured < pebbleCachePercentMin || configured > pebbleCachePercentMax {
+		return defaultPebbleCachePercent
+	}
+	return configured
+}
+
 var metaLastCommitTSBytes = []byte(metaLastCommitTS)
 var metaMinRetainedTSBytes = []byte(metaMinRetainedTS)
 var metaPendingMinRetainedTSBytes = []byte(metaPendingMinRetainedTS)
@@ -358,11 +393,9 @@ func NewPebbleStore(dir string, opts ...PebbleStoreOption) (MVCCStore, error) {
 	s.db = db
 	s.cache = cache
 
-	// cleanupOnInitFail closes the just-opened DB and releases our creator
-	// reference on the cache so that a partially corrupt store (where the
-	// metadata scans below fail) does not leak the ~256 MiB cache
-	// allocation. Also nil's out the store pointers so a zero-valued
-	// pebbleStore does not linger with stale refs.
+	// cleanupOnInitFail closes the just-opened DB and releases this open's
+	// borrowed cache reference when a metadata scan fails. Also nil out the
+	// store pointers so a zero-valued pebbleStore does not retain stale refs.
 	cleanupOnInitFail := func() {
 		_ = db.Close()
 		cache.Unref()
@@ -3164,8 +3197,8 @@ func writeNativeSnapshotToTempDir(r io.Reader, tmpDir string, ts uint64) error {
 		return errors.WithStack(err)
 	}
 	// The tmpDB is discarded once swapInTempDB renames its directory over
-	// s.dir (which reopens with a fresh cache), so release our creator ref
-	// as soon as we're done writing to avoid leaking per-snapshot caches.
+	// s.dir, so release this temporary open's borrowed cache reference when
+	// the helper returns. The destination reopen borrows its own reference.
 	defer cache.Unref()
 
 	if err := restoreBatchLoopInto(r, tmpDB); err != nil {
@@ -3215,8 +3248,8 @@ func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash3
 		return "", errors.WithStack(err)
 	}
 	// As in writeNativeSnapshotToTempDir, the tmpDB is discarded once the
-	// directory is renamed, so drop our creator ref when this helper
-	// returns. swapInTempDB reopens with a fresh cache.
+	// directory is renamed, so drop this temporary open's borrowed cache
+	// reference when the helper returns. swapInTempDB borrows its own ref.
 	defer cache.Unref()
 	cleanupTmp := func() {
 		_ = tmpDB.Close()
@@ -3388,11 +3421,10 @@ func (s *pebbleStore) Close() error {
 	s.dbMu.Lock()
 	defer s.dbMu.Unlock()
 	err := s.db.Close()
-	// Release our creator reference on the block cache so the memory is
-	// freed once pebble has also dropped its internal reference (which
-	// Close does). Safe to call after a Close error: Unref is
-	// idempotent-friendly in that s.cache is only nilled out here, and
-	// Close() is not expected to be called twice.
+	// Release this store's borrowed block-cache reference after Pebble has
+	// dropped its internal handle. The process base reference keeps the shared
+	// cache alive for other stores; nil prevents a second Close from unrefing
+	// this store's reference twice.
 	if s.cache != nil {
 		s.cache.Unref()
 		s.cache = nil
diff --git a/store/lsm_store_env_test.go b/store/lsm_store_env_test.go
index fbe2a6732..aba6f9376 100644
--- a/store/lsm_store_env_test.go
+++ b/store/lsm_store_env_test.go
@@ -1,6 +1,9 @@
 package store
 
 import (
+	"bytes"
+	"context"
+	"sync"
 	"testing"
 
 	"github.com/cockroachdb/pebble/v2"
@@ -9,8 +12,8 @@ import (
 
 // setSmallPebbleCacheForTest swaps the package-level pebbleCacheBytes value
 // to 16 MiB for the duration of a single test and restores it during
-// t.Cleanup. The real override happens in init() from
-// ELASTICKV_PEBBLE_CACHE_MB; tests use this helper without relying on process
+// t.Cleanup. The real sizing happens in init() from the node memory budget and
+// ELASTICKV_PEBBLE_CACHE_*; tests use this helper without relying on process
 // env state.
 func setSmallPebbleCacheForTest(t *testing.T) {
 	t.Helper()
@@ -19,51 +22,94 @@ func setSmallPebbleCacheForTest(t *testing.T) {
 	t.Cleanup(func() { pebbleCacheBytes = prev })
 }
 
-// TestPebbleCacheEnvOverride covers the ELASTICKV_PEBBLE_CACHE_MB parsing
-// contract directly against resolvePebbleCacheBytes, which is what init()
-// calls. We deliberately avoid mutating os.Environ here so parallel test
-// binaries remain isolated.
+// TestPebbleCacheEnvOverride covers the absolute and percentage-based cache
+// sizing contract directly against resolvePebbleCacheBytes, which is what
+// init() calls. We deliberately avoid mutating os.Environ here so parallel
+// test binaries remain isolated.
 func TestPebbleCacheEnvOverride(t *testing.T) {
+	const memoryBudgetBytes int64 = 4000 << 20
+
 	t.Run("valid value is parsed and converted to bytes", func(t *testing.T) {
-		got := resolvePebbleCacheBytes("64")
+		got := resolvePebbleCacheBytes("64", "", memoryBudgetBytes)
 		require.Equal(t, int64(64)<<20, got)
 	})
 
-	t.Run("empty string falls back to default", func(t *testing.T) {
-		require.Equal(t, defaultPebbleCacheBytes, resolvePebbleCacheBytes(""))
+	t.Run("empty string uses twenty five percent of memory budget", func(t *testing.T) {
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("", "", memoryBudgetBytes))
+	})
+
+	t.Run("explicit percentage is applied", func(t *testing.T) {
+		require.Equal(t, int64(400)<<20, resolvePebbleCacheBytes("", "10", memoryBudgetBytes))
+	})
+
+	t.Run("absolute override wins over percentage", func(t *testing.T) {
+		require.Equal(t, int64(64)<<20, resolvePebbleCacheBytes("64", "10", memoryBudgetBytes))
 	})
 
-	t.Run("garbage input falls back to default", func(t *testing.T) {
-		require.Equal(t, defaultPebbleCacheBytes, resolvePebbleCacheBytes("garbage"))
+	t.Run("garbage absolute input falls back to percentage default", func(t *testing.T) {
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("garbage", "", memoryBudgetBytes))
 	})
 
-	t.Run("zero is rejected and default is applied", func(t *testing.T) {
+	t.Run("garbage percentage falls back to twenty five percent", func(t *testing.T) {
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("", "garbage", memoryBudgetBytes))
+	})
+
+	t.Run("percentage boundaries are enforced", func(t *testing.T) {
+		require.Equal(t, int64(40)<<20, resolvePebbleCacheBytes("", "1", memoryBudgetBytes))
+		require.Equal(t, int64(3600)<<20, resolvePebbleCacheBytes("", "90", memoryBudgetBytes))
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("", "0", memoryBudgetBytes))
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("", "91", memoryBudgetBytes))
+	})
+
+	t.Run("zero is rejected and percentage default is applied", func(t *testing.T) {
 		// "0" is below the 8 MiB floor, so we fall back to the default
 		// rather than silently clamping. Documented behaviour: only
 		// values inside [8, 65536] MiB are accepted; everything else
 		// falls back.
-		require.Equal(t, defaultPebbleCacheBytes, resolvePebbleCacheBytes("0"))
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("0", "", memoryBudgetBytes))
 	})
 
-	t.Run("below floor falls back to default", func(t *testing.T) {
-		require.Equal(t, defaultPebbleCacheBytes, resolvePebbleCacheBytes("4"))
+	t.Run("below floor falls back to percentage default", func(t *testing.T) {
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("4", "", memoryBudgetBytes))
 	})
 
 	t.Run("at floor is accepted", func(t *testing.T) {
-		require.Equal(t, int64(8)<<20, resolvePebbleCacheBytes("8"))
+		require.Equal(t, int64(8)<<20, resolvePebbleCacheBytes("8", "", memoryBudgetBytes))
 	})
 
 	t.Run("at ceiling is accepted", func(t *testing.T) {
-		require.Equal(t, int64(65536)<<20, resolvePebbleCacheBytes("65536"))
+		require.Equal(t, int64(65536)<<20, resolvePebbleCacheBytes("65536", "", memoryBudgetBytes))
 	})
 
-	t.Run("above ceiling falls back to default", func(t *testing.T) {
-		require.Equal(t, defaultPebbleCacheBytes, resolvePebbleCacheBytes("65537"))
+	t.Run("above ceiling falls back to percentage default", func(t *testing.T) {
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("65537", "", memoryBudgetBytes))
 	})
 
-	t.Run("negative falls back to default", func(t *testing.T) {
-		require.Equal(t, defaultPebbleCacheBytes, resolvePebbleCacheBytes("-1"))
+	t.Run("negative falls back to percentage default", func(t *testing.T) {
+		require.Equal(t, int64(1000)<<20, resolvePebbleCacheBytes("-1", "", memoryBudgetBytes))
 	})
+
+	t.Run("unknown memory budget uses fixed fallback", func(t *testing.T) {
+		require.Equal(t, defaultPebbleCacheBytes, resolvePebbleCacheBytes("", "", 0))
+	})
+
+	t.Run("percentage result observes cache floor", func(t *testing.T) {
+		require.Equal(t, int64(8)<<20, resolvePebbleCacheBytes("", "", 16<<20))
+	})
+}
+
+func TestMemoryBudgetHelpers(t *testing.T) {
+	require.Equal(t, int64(512), minPositiveMemoryBytes(0, 1024, -1, 512, 2048))
+	require.Equal(t, int64(0), minPositiveMemoryBytes(0, -1))
+	require.Equal(t, int64(1024), parseMemoryLimitBytes([]byte("1024\n")))
+	require.Equal(t, int64(0), parseMemoryLimitBytes([]byte("max\n")))
+	require.Equal(t, int64(0), parseMemoryLimitBytes([]byte("invalid")))
+	require.Equal(t, []string{
+		"/sys/fs/cgroup/system.slice/elastickv.service/memory.max",
+	}, cgroupMemoryLimitPaths([]byte("0::/system.slice/elastickv.service\n")))
+	require.Equal(t, []string{
+		"/sys/fs/cgroup/memory/docker/abc/memory.limit_in_bytes",
+	}, cgroupMemoryLimitPaths([]byte("5:cpu,memory:/docker/abc\n")))
 }
 
 // TestSetSmallPebbleCacheForTestRestores verifies the helper reinstates the
@@ -102,12 +148,44 @@ func TestDefaultPebbleOptionsSharesProcessCache(t *testing.T) {
 	require.Same(t, cache1, opts2.Cache)
 }
 
+func TestDefaultPebbleOptionsSharesProcessCacheConcurrently(t *testing.T) {
+	setSmallPebbleCacheForTest(t)
+
+	const borrowers = 16
+	caches := make(chan *pebble.Cache, borrowers)
+	var wg sync.WaitGroup
+	wg.Add(borrowers)
+	for range borrowers {
+		go func() {
+			defer wg.Done()
+			_, cache := defaultPebbleOptionsWithCache()
+			caches <- cache
+		}()
+	}
+	wg.Wait()
+	close(caches)
+
+	var shared *pebble.Cache
+	for cache := range caches {
+		if shared == nil {
+			shared = cache
+		}
+		require.Same(t, shared, cache)
+		cache.Unref()
+	}
+}
+
 func TestNewPebbleStoreSharesProcessCache(t *testing.T) {
 	setSmallPebbleCacheForTest(t)
 
 	s1, err := NewPebbleStore(t.TempDir())
 	require.NoError(t, err)
-	defer s1.Close()
+	closed1 := false
+	t.Cleanup(func() {
+		if !closed1 {
+			_ = s1.Close()
+		}
+	})
 	ps1, ok := s1.(*pebbleStore)
 	require.True(t, ok)
 
@@ -121,6 +199,49 @@ func TestNewPebbleStoreSharesProcessCache(t *testing.T) {
 	require.Same(t, ps1.cache, ps2.cache)
 	require.Equal(t, int64(16)<<20, ps1.BlockCacheCapacityBytes())
 	require.Equal(t, ps1.BlockCacheCapacityBytes(), ps2.BlockCacheCapacityBytes())
+
+	require.NoError(t, s1.Close())
+	closed1 = true
+	require.NoError(t, s2.PutAt(context.Background(), []byte("still-open"), []byte("value"), 1, 0))
+	value, err := s2.GetAt(context.Background(), []byte("still-open"), 1)
+	require.NoError(t, err)
+	require.Equal(t, []byte("value"), value)
+}
+
+func TestPebbleStoreRestoreKeepsProcessCache(t *testing.T) {
+	setSmallPebbleCacheForTest(t)
+	ctx := context.Background()
+
+	src := NewMVCCStore()
+	require.NoError(t, src.PutAt(ctx, []byte("key"), []byte("value"), 1, 0))
+	snapshot, err := src.Snapshot()
+	require.NoError(t, err)
+	var streaming bytes.Buffer
+	_, err = snapshot.WriteTo(&streaming)
+	require.NoError(t, err)
+	require.NoError(t, snapshot.Close())
+
+	dst, err := NewPebbleStore(t.TempDir())
+	require.NoError(t, err)
+	defer dst.Close()
+	ps, ok := dst.(*pebbleStore)
+	require.True(t, ok)
+	cache := ps.cache
+
+	require.NoError(t, dst.Restore(bytes.NewReader(streaming.Bytes())))
+	require.Same(t, cache, ps.cache, "streaming restore must reopen on the process cache")
+
+	nativeSnapshot, err := dst.Snapshot()
+	require.NoError(t, err)
+	var native bytes.Buffer
+	_, err = nativeSnapshot.WriteTo(&native)
+	require.NoError(t, err)
+	require.NoError(t, nativeSnapshot.Close())
+	require.NoError(t, dst.Restore(bytes.NewReader(native.Bytes())))
+	require.Same(t, cache, ps.cache, "native restore must reopen on the process cache")
+
+	require.NoError(t, dst.Restore(bytes.NewReader(nil)))
+	require.Same(t, cache, ps.cache, "empty restore must reopen on the process cache")
 }
 
 // newPebbleStoreWithFSMApplyWriteOptsForTest constructs a pebbleStore
diff --git a/store/pebble_memory_budget.go b/store/pebble_memory_budget.go
new file mode 100644
index 000000000..b9c12e076
--- /dev/null
+++ b/store/pebble_memory_budget.go
@@ -0,0 +1,67 @@
+package store
+
+import (
+	"math"
+	"path/filepath"
+	"runtime/debug"
+	"strconv"
+	"strings"
+)
+
+const cgroupFieldCount = 3
+
+// effectiveMemoryBudgetBytes returns the smallest process or platform memory
+// limit visible to the node. GOMEMLIMIT is included because production may
+// deliberately give elastickv a smaller budget than its container or host.
+func effectiveMemoryBudgetBytes() int64 {
+	runtimeLimit := debug.SetMemoryLimit(-1)
+	if runtimeLimit <= 0 || runtimeLimit == math.MaxInt64 {
+		runtimeLimit = 0
+	}
+	return minPositiveMemoryBytes(runtimeLimit, platformMemoryBudgetBytes())
+}
+
+func minPositiveMemoryBytes(values ...int64) int64 {
+	var minimum int64
+	for _, value := range values {
+		if value <= 0 || minimum != 0 && value >= minimum {
+			continue
+		}
+		minimum = value
+	}
+	return minimum
+}
+
+func parseMemoryLimitBytes(raw []byte) int64 {
+	value := strings.TrimSpace(string(raw))
+	if value == "" || value == "max" {
+		return 0
+	}
+	parsed, err := strconv.ParseUint(value, 10, 64)
+	if err != nil || parsed > math.MaxInt64 {
+		return 0
+	}
+	return int64(parsed)
+}
+
+func cgroupMemoryLimitPaths(procSelfCgroup []byte) []string {
+	var paths []string
+	for _, line := range strings.Split(strings.TrimSpace(string(procSelfCgroup)), "\n") {
+		fields := strings.SplitN(line, ":", cgroupFieldCount)
+		if len(fields) != cgroupFieldCount {
+			continue
+		}
+		relative := strings.TrimPrefix(filepath.Clean(fields[2]), string(filepath.Separator))
+		if fields[0] == "0" && fields[1] == "" {
+			paths = append(paths, filepath.Join("/sys/fs/cgroup", relative, "memory.max"))
+			continue
+		}
+		for _, controller := range strings.Split(fields[1], ",") {
+			if controller == "memory" {
+				paths = append(paths, filepath.Join("/sys/fs/cgroup/memory", relative, "memory.limit_in_bytes"))
+				break
+			}
+		}
+	}
+	return paths
+}
diff --git a/store/pebble_memory_darwin.go b/store/pebble_memory_darwin.go
new file mode 100644
index 000000000..48e675013
--- /dev/null
+++ b/store/pebble_memory_darwin.go
@@ -0,0 +1,17 @@
+//go:build darwin
+
+package store
+
+import (
+	"math"
+
+	"golang.org/x/sys/unix"
+)
+
+func platformMemoryBudgetBytes() int64 {
+	total, err := unix.SysctlUint64("hw.memsize")
+	if err != nil || total > math.MaxInt64 {
+		return 0
+	}
+	return int64(total)
+}
diff --git a/store/pebble_memory_linux.go b/store/pebble_memory_linux.go
new file mode 100644
index 000000000..1cbe701c2
--- /dev/null
+++ b/store/pebble_memory_linux.go
@@ -0,0 +1,47 @@
+//go:build linux
+
+package store
+
+import (
+	"math"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+var rootCgroupMemoryLimitPaths = []string{
+	"/sys/fs/cgroup/memory.max",
+	"/sys/fs/cgroup/memory/memory.limit_in_bytes",
+}
+
+func platformMemoryBudgetBytes() int64 {
+	return minPositiveMemoryBytes(systemMemoryBytes(), cgroupMemoryLimitBytes())
+}
+
+func systemMemoryBytes() int64 {
+	var info unix.Sysinfo_t
+	if err := unix.Sysinfo(&info); err != nil {
+		return 0
+	}
+	total := uint64(info.Totalram) * uint64(info.Unit)
+	if total > math.MaxInt64 {
+		return math.MaxInt64
+	}
+	return int64(total)
+}
+
+func cgroupMemoryLimitBytes() int64 {
+	paths := rootCgroupMemoryLimitPaths
+	if procSelfCgroup, err := os.ReadFile("/proc/self/cgroup"); err == nil {
+		paths = append(cgroupMemoryLimitPaths(procSelfCgroup), paths...)
+	}
+	var limit int64
+	for _, path := range paths {
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			continue
+		}
+		limit = minPositiveMemoryBytes(limit, parseMemoryLimitBytes(raw))
+	}
+	return limit
+}
diff --git a/store/pebble_memory_other.go b/store/pebble_memory_other.go
new file mode 100644
index 000000000..8e97627a0
--- /dev/null
+++ b/store/pebble_memory_other.go
@@ -0,0 +1,7 @@
+//go:build !darwin && !linux
+
+package store
+
+func platformMemoryBudgetBytes() int64 {
+	return 0
+}

From e10bfa55a40d84321ee06a0239a9ff7719375907 Mon Sep 17 00:00:00 2001
From: bootjp <contact@bootjp.me>
Date: Sat, 18 Jul 2026 20:07:52 +0900
Subject: [PATCH 4/6] store: remove redundant memory conversion

---
 store/pebble_memory_linux.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/store/pebble_memory_linux.go b/store/pebble_memory_linux.go
index 1cbe701c2..b01beb9c3 100644
--- a/store/pebble_memory_linux.go
+++ b/store/pebble_memory_linux.go
@@ -23,7 +23,7 @@ func systemMemoryBytes() int64 {
 	if err := unix.Sysinfo(&info); err != nil {
 		return 0
 	}
-	total := uint64(info.Totalram) * uint64(info.Unit)
+	total := info.Totalram * uint64(info.Unit)
 	if total > math.MaxInt64 {
 		return math.MaxInt64
 	}

From f84ecbae448915158fd1f812e9343473c35d82b4 Mon Sep 17 00:00:00 2001
From: bootjp <contact@bootjp.me>
Date: Sun, 19 Jul 2026 18:45:46 +0900
Subject: [PATCH 5/6] Harden shared cache memory budgeting

---
 .../2026_06_12_proposed_scaling_roadmap.md    |  7 ++-
 .../2026_06_23_proposed_scaling_roadmap.md    | 13 ++++--
 store/lsm_store.go                            |  6 +--
 store/lsm_store_env_test.go                   | 11 +++++
 store/pebble_memory_budget.go                 | 46 ++++++++++++++-----
 5 files changed, 61 insertions(+), 22 deletions(-)

diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md
index a32acbbca..2afe9bfe9 100644
--- a/docs/design/2026_06_12_proposed_scaling_roadmap.md
+++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md
@@ -110,8 +110,11 @@ Multi-region blockers:
 ### 2.3 Storage tier (Pebble)
 
 `store/lsm_store.go`: per-shard `pebble.Open`, process-wide shared block cache
-sized to 25% of the node's effective memory budget by default and shared by all
-stores in the process. `ELASTICKV_PEBBLE_CACHE_PERCENT` changes that fraction;
+sized to 25% of the node's hard memory capacity by default and shared by all
+stores in the process. Linux uses the minimum positive cgroup limit found from
+the process leaf through every ancestor, bounded by physical RAM. `GOMEMLIMIT`
+is not used as a total RSS budget because the Pebble cache is off-heap.
+`ELASTICKV_PEBBLE_CACHE_PERCENT` changes the hard-capacity fraction;
 `ELASTICKV_PEBBLE_CACHE_MB` is the absolute-capacity override. WAL sync via
 `ELASTICKV_FSM_SYNC_MODE` (default `pebble.Sync` on FSM apply; `nosync` opt-in).
 
diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md
index e53d01a19..4b6634c3a 100644
--- a/docs/design/2026_06_23_proposed_scaling_roadmap.md
+++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md
@@ -41,9 +41,11 @@ What bounds a single elastickv deployment today:
 2. **Per-group memory.** Each Raft group owns a private Pebble store, and
    `NewPebbleStore` → `defaultPebbleOptionsWithCache` now borrows one
    process-wide shared block cache through `processPebbleCacheRef`, defaulting
-   to 25% of the smallest discovered GOMEMLIMIT, cgroup limit, or physical RAM
-   size. N groups on a node therefore share one block-cache LRU instead of
-   reserving an independent fixed-size cache per group.
+   to 25% of the smallest discovered cgroup ancestor limit or physical RAM
+   size. `GOMEMLIMIT` remains a Go heap soft limit and is not treated as total
+   RSS capacity for the off-heap cache. N groups on a node therefore share one
+   block-cache LRU instead of reserving an independent fixed-size cache per
+   group.
    Per-group memtables, WALs, and compaction budgets are still independent.
 
 3. **Leader concentration.** Leadership of each group is elected
@@ -306,8 +308,9 @@ process-wide shared `pebble.Cache` via `processPebbleCacheRef`, and
 `NewPebbleStore` / restore reopen paths hold one explicit store/open reference
 that is released on close or reopen. `ELASTICKV_PEBBLE_CACHE_MB` now sizes the
 node-level shared cache rather than a per-store cache; when it is unset,
-`ELASTICKV_PEBBLE_CACHE_PERCENT` selects the fraction of the node's effective
-memory budget (25% by default).
+`ELASTICKV_PEBBLE_CACHE_PERCENT` selects the fraction of the node's hard
+memory capacity (25% by default). Linux takes the minimum positive cgroup
+limit from the process leaf through every ancestor and physical RAM.
 
 **Remaining problem.** Each group's store still owns independent memtables,
 WALs, flush scheduling, and compaction concurrency. That still caps how many
diff --git a/store/lsm_store.go b/store/lsm_store.go
index 1e2b58a24..ae8bbd4b6 100644
--- a/store/lsm_store.go
+++ b/store/lsm_store.go
@@ -75,9 +75,9 @@ const (
 	pebbleCacheMBEnv = "ELASTICKV_PEBBLE_CACHE_MB"
 
 	// pebbleCachePercentEnv controls the percentage (1..90) of the node's
-	// effective memory budget reserved for the shared cache when no absolute
-	// MiB override is set. The effective budget is the smallest discovered
-	// GOMEMLIMIT, cgroup limit, or physical-memory size.
+	// hard memory capacity reserved for the shared cache when no absolute MiB
+	// override is set. Linux uses the smallest positive cgroup ancestor limit
+	// or physical-memory size; GOMEMLIMIT remains a Go heap soft limit.
 	pebbleCachePercentEnv = "ELASTICKV_PEBBLE_CACHE_PERCENT"
 
 	defaultPebbleCachePercent = 25
diff --git a/store/lsm_store_env_test.go b/store/lsm_store_env_test.go
index aba6f9376..9ba1d7402 100644
--- a/store/lsm_store_env_test.go
+++ b/store/lsm_store_env_test.go
@@ -106,10 +106,21 @@ func TestMemoryBudgetHelpers(t *testing.T) {
 	require.Equal(t, int64(0), parseMemoryLimitBytes([]byte("invalid")))
 	require.Equal(t, []string{
 		"/sys/fs/cgroup/system.slice/elastickv.service/memory.max",
+		"/sys/fs/cgroup/system.slice/memory.max",
+		"/sys/fs/cgroup/memory.max",
 	}, cgroupMemoryLimitPaths([]byte("0::/system.slice/elastickv.service\n")))
 	require.Equal(t, []string{
 		"/sys/fs/cgroup/memory/docker/abc/memory.limit_in_bytes",
+		"/sys/fs/cgroup/memory/docker/memory.limit_in_bytes",
+		"/sys/fs/cgroup/memory/memory.limit_in_bytes",
 	}, cgroupMemoryLimitPaths([]byte("5:cpu,memory:/docker/abc\n")))
+	require.Equal(t, []string{
+		"/sys/fs/cgroup/kubepods/pod-1/container-1/memory.max",
+		"/sys/fs/cgroup/kubepods/pod-1/memory.max",
+		"/sys/fs/cgroup/kubepods/memory.max",
+		"/sys/fs/cgroup/memory.max",
+	}, cgroupAncestorLimitPaths("/sys/fs/cgroup", "kubepods/pod-1/container-1", "memory.max"))
+	require.Empty(t, cgroupAncestorLimitPaths("/sys/fs/cgroup", "../escape", "memory.max"))
 }
 
 // TestSetSmallPebbleCacheForTestRestores verifies the helper reinstates the
diff --git a/store/pebble_memory_budget.go b/store/pebble_memory_budget.go
index b9c12e076..55ecea26e 100644
--- a/store/pebble_memory_budget.go
+++ b/store/pebble_memory_budget.go
@@ -3,22 +3,17 @@ package store
 import (
 	"math"
 	"path/filepath"
-	"runtime/debug"
 	"strconv"
 	"strings"
 )
 
 const cgroupFieldCount = 3
 
-// effectiveMemoryBudgetBytes returns the smallest process or platform memory
-// limit visible to the node. GOMEMLIMIT is included because production may
-// deliberately give elastickv a smaller budget than its container or host.
+// effectiveMemoryBudgetBytes returns the hard platform memory capacity visible
+// to the node. GOMEMLIMIT is a Go heap soft limit, not a total RSS budget, and
+// Pebble's manually allocated block cache must not be sized as a fraction of it.
 func effectiveMemoryBudgetBytes() int64 {
-	runtimeLimit := debug.SetMemoryLimit(-1)
-	if runtimeLimit <= 0 || runtimeLimit == math.MaxInt64 {
-		runtimeLimit = 0
-	}
-	return minPositiveMemoryBytes(runtimeLimit, platformMemoryBudgetBytes())
+	return platformMemoryBudgetBytes()
 }
 
 func minPositiveMemoryBytes(values ...int64) int64 {
@@ -51,17 +46,44 @@ func cgroupMemoryLimitPaths(procSelfCgroup []byte) []string {
 		if len(fields) != cgroupFieldCount {
 			continue
 		}
-		relative := strings.TrimPrefix(filepath.Clean(fields[2]), string(filepath.Separator))
+		cleaned := filepath.Clean(string(filepath.Separator) + fields[2])
+		relative := strings.TrimPrefix(cleaned, string(filepath.Separator))
 		if fields[0] == "0" && fields[1] == "" {
-			paths = append(paths, filepath.Join("/sys/fs/cgroup", relative, "memory.max"))
+			paths = append(paths, cgroupAncestorLimitPaths("/sys/fs/cgroup", relative, "memory.max")...)
 			continue
 		}
 		for _, controller := range strings.Split(fields[1], ",") {
 			if controller == "memory" {
-				paths = append(paths, filepath.Join("/sys/fs/cgroup/memory", relative, "memory.limit_in_bytes"))
+				paths = append(paths, cgroupAncestorLimitPaths(
+					"/sys/fs/cgroup/memory",
+					relative,
+					"memory.limit_in_bytes",
+				)...)
 				break
 			}
 		}
 	}
 	return paths
 }
+
+func cgroupAncestorLimitPaths(root, relative, filename string) []string {
+	root = filepath.Clean(root)
+	dir := filepath.Join(root, relative)
+	rel, err := filepath.Rel(root, dir)
+	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+		return nil
+	}
+
+	var paths []string
+	for {
+		paths = append(paths, filepath.Join(dir, filename))
+		if dir == root {
+			return paths
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			return paths
+		}
+		dir = parent
+	}
+}

From b4ab1a666c890ef2294b38eee34d074ad057be43 Mon Sep 17 00:00:00 2001
From: bootjp <contact@bootjp.me>
Date: Sun, 19 Jul 2026 22:41:55 +0900
Subject: [PATCH 6/6] store: preserve cache test override

---
 store/lsm_store_env_test.go | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/store/lsm_store_env_test.go b/store/lsm_store_env_test.go
index 9ba1d7402..0fa262f36 100644
--- a/store/lsm_store_env_test.go
+++ b/store/lsm_store_env_test.go
@@ -10,18 +10,22 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// setSmallPebbleCacheForTest swaps the package-level pebbleCacheBytes value
-// to 16 MiB for the duration of a single test and restores it during
-// t.Cleanup. The real sizing happens in init() from the node memory budget and
-// ELASTICKV_PEBBLE_CACHE_*; tests use this helper without relying on process
-// env state.
-func setSmallPebbleCacheForTest(t *testing.T) {
+// setPebbleCacheBytesForTest swaps the package-level cache capacity for one
+// test. defaultPebbleOptionsWithCache replaces the process cache lazily when
+// it observes a different capacity, so callers retain the production sharing
+// and reference-counting path.
+func setPebbleCacheBytesForTest(t *testing.T, capacity int64) {
 	t.Helper()
 	prev := pebbleCacheBytes
-	pebbleCacheBytes = 16 << 20
+	pebbleCacheBytes = capacity
 	t.Cleanup(func() { pebbleCacheBytes = prev })
 }
 
+func setSmallPebbleCacheForTest(t *testing.T) {
+	t.Helper()
+	setPebbleCacheBytesForTest(t, 16<<20)
+}
+
 // TestPebbleCacheEnvOverride covers the absolute and percentage-based cache
 // sizing contract directly against resolvePebbleCacheBytes, which is what
 // init() calls. We deliberately avoid mutating os.Environ here so parallel