From 56c981fe98632ffe38a1e0a66e65dcc8d06ae1f2 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Tue, 21 Apr 2026 12:23:28 -0600 Subject: [PATCH 01/21] cache: bump to go 1.24, fix stale-at-epoch, tighten Clean - Bump go.mod from 1.18 to 1.24. The code has required at least 1.19 since the switch to generic atomic types (atomic.Pointer[T], atomic.Int64, atomic.Uint32). 1.24 also unlocks maps.Copy, which replaces the hand-rolled copy loop in promote. - newStale: when the main entry's expiry is 0 (infinite cache) or -1 (immediate expiry), base the stale's lifespan on now() instead of producing a stale born at the Unix epoch. This was a latent bug: Get paths typically regenerated the stale via maybeNewStale before it was observed, but the initial stale set by Swap/Set was wrong. - Clean: call e.del() directly instead of c.Delete(k). c.each already promotes when necessary, so every entry the callback sees lives in the read map and needs no dirty-map bookkeeping or per-key relock. - Document that Expire is a no-op on in-flight loads for Cache, Item, and Set. --- cache/cache.go | 33 ++++++++++++++++++++++++++++----- go.mod | 2 +- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 0e6d5af..f598979 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -16,6 +16,7 @@ package cache import ( + "maps" "sync" "sync/atomic" "time" @@ -325,6 +326,10 @@ func (c *Cache[K, V]) Delete(k K) (V, error, KeyState) { // Expire sets a stored value to expire immediately, meaning the next Get will // be a miss. If stale values are enabled, the next Get will trigger the miss // function but still allow the now-stale value to be returned. +// +// Expire only affects entries that have finished loading. Calling Expire for a +// key whose miss function is still running is a no-op; the in-flight load will +// complete with its normal TTL and is not canceled or shortened. func (c *Cache[K, V]) Expire(k K) { e := c.tryLoadEnt(k, nil) if l := e.load(); l != nil && l.finalized() { @@ -377,11 +382,15 @@ func (c *Cache[K, V]) Clean() { return } now := now() - c.each(func(k K, e *ent[V]) bool { + c.each(func(_ K, e *ent[V]) bool { if l := e.load(); l != nil && l.finalized() { expires := l.expires.Load() if expires != 0 && now > expires+int64(c.cfg.maxStaleAge) { - c.Delete(k) + // c.each promotes when necessary, so every entry + // we see lives in the read map. e.del alone is + // sufficient; no dirty bookkeeping or per-key + // relock is required. + e.del() } } return true @@ -606,9 +615,7 @@ func (c *Cache[K, V]) promote() { } keep = make(map[K]*ent[V], len(keep)+len(c.dirty)) - for k, e := range c.dirty { - keep[k] = e - } + maps.Copy(keep, c.dirty) c.dirty = nil outer: @@ -708,10 +715,18 @@ func (e *ent[V]) maybeNewStale(age time.Duration) *stale[V] { } // Actually returns the stale; age must be non-zero. +// +// expires is the main entry's expiry nano. If it is <= 0 (either 0 meaning +// the main entry never expires, or -1 meaning it expired immediately), we +// base the stale's lifespan on now so the stale is not born expired at the +// Unix epoch. func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { if age < 0 { return &stale[V]{v: v} } + if expires <= 0 { + expires = now() + } return &stale[V]{v, expires + int64(age)} } @@ -851,6 +866,10 @@ func (i *Item[V]) Delete() (V, error, KeyState) { // Expire sets the item to expire immediately, meaning the next call to Get // will be a miss. If stale values are enabled, the next Get will trigger the // miss function but still allow the now-stale value to be returned. +// +// Expire only affects an item that has finished loading. Calling Expire while +// the miss function is still running is a no-op; the in-flight load will +// complete with its normal TTL and is not canceled or shortened. func (i *Item[V]) Expire() { i.c.Expire(struct{}{}) } @@ -943,6 +962,10 @@ func (s *Set[K]) Delete(k K) (error, KeyState) { // Expire sets the key to expire immediately, meaning the next call to Get will // be a miss. If stale keys are enabled, the next Get will trigger the miss // function but still allow a now-stale nil error to be returned. +// +// Expire only affects a key whose load has finished. Calling Expire while the +// miss function is still running is a no-op; the in-flight load will complete +// with its normal TTL and is not canceled or shortened. func (s *Set[K]) Expire(k K) { s.c.Expire(k) } diff --git a/go.mod b/go.mod index da78d2d..39644da 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/twmb/go-cache -go 1.18 +go 1.24 retract v1.1.0 // The newly added API cannot work. From d6e9fc365f6a7a18caf861d3a00013eceaebc420 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Tue, 21 Apr 2026 13:06:55 -0600 Subject: [PATCH 02/21] cache: expand test suite; fix data race in Get's post-miss return Test coverage climbs from 84.3% to 100% of statements. The new tests either target concretely uncovered behavior (Item/Set wrappers, autoclean lifecycle, Clean no-op on MaxStaleAge<0) or target race windows in the existing read/dirty implementation (Swap-vs-in-flight-Get, Delete-during-load, Expire-no-op-during-load, maxAge=0 collapsing, the promote CAS-retry path, Swap observing the promotingDelete sentinel, Get's unlocked-miss / locked-hit interleaving, CompareAndSwap against a just-promoted entry). Single -count=1 -race runs land 100% coverage reliably on this machine; the stress test driving the tightest windows runs for ~2s against many goroutines. The Swap-vs-in-flight-Get test caught a real data race (reported by -race). After Get's final e.get, the original code returned l.v without synchronizing with any writer: concurrent Swap's defer writes l.v under l.mu and calls l.wg.Done, but Get was only waiting on whatever loading e.p pointed to at the time, which could be Swap's replacement (a different loading). Add l.wg.Wait() before returning l.v to establish the happens-before edge with whoever finalized l (either setve or Swap's defer). New test files: - wrappers_test.go: every Item and Set method - autoclean_test.go: AutoCleanInterval goroutine lifecycle and StopAutoClean idempotency - concurrency_test.go: race-targeted tests and the miss/stale correctness properties called out in the trie port plan --- cache/autoclean_test.go | 116 ++++++ cache/cache.go | 8 + cache/concurrency_test.go | 730 ++++++++++++++++++++++++++++++++++++++ cache/wrappers_test.go | 289 +++++++++++++++ 4 files changed, 1143 insertions(+) create mode 100644 cache/autoclean_test.go create mode 100644 cache/concurrency_test.go create mode 100644 cache/wrappers_test.go diff --git a/cache/autoclean_test.go b/cache/autoclean_test.go new file mode 100644 index 0000000..ab2e5d1 --- /dev/null +++ b/cache/autoclean_test.go @@ -0,0 +1,116 @@ +package cache + +import ( + "sync/atomic" + "testing" + "time" +) + +func TestAutoClean_RunsAndCleansExpired(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(2*time.Millisecond), + ) + defer c.StopAutoClean() + + c.Set("a", 1) + if _, _, s := c.TryGet("a"); !s.IsHit() { + t.Fatalf("immediately after Set: want Hit") + } + + // Wait long enough for the ticker to tick at least a few times. + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if _, _, s := c.TryGet("a"); s.IsMiss() { + return // AutoClean fired and removed the expired entry. + } + time.Sleep(2 * time.Millisecond) + } + t.Fatal("AutoClean never removed the expired entry") +} + +func TestAutoClean_DisabledWhenStaleAgeNegative(t *testing.T) { + // AutoCleanInterval is ignored when MaxStaleAge < 0 (stales kept forever). + // We verify no goroutine is spawned by confirming StopAutoClean is a no-op + // on a cache whose quitClean channel was never created. + c := New[string, int]( + MaxAge(time.Millisecond), + MaxStaleAge(-1), + AutoCleanInterval(time.Millisecond), + ) + if c.quitClean != nil { + t.Fatal("quitClean channel should not be allocated when MaxStaleAge<0") + } + c.StopAutoClean() // no-op must not panic +} + +func TestAutoClean_StopIdempotent(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(time.Millisecond), + ) + c.StopAutoClean() + c.StopAutoClean() // sync.Once guards the close; second call must be safe. +} + +func TestAutoClean_StopOnCacheWithoutGoroutine(t *testing.T) { + // A cache created without AutoCleanInterval has no goroutine and no + // quitClean channel. StopAutoClean must still be safe to call. + c := New[string, int]() + c.StopAutoClean() +} + +func TestAutoClean_ZeroIntervalNoGoroutine(t *testing.T) { + // autoCleanInterval == 0 means disabled; no goroutine, no channel. + c := New[string, int](MaxAge(time.Second)) + if c.quitClean != nil { + t.Fatal("quitClean channel should not be allocated with zero interval") + } + c.StopAutoClean() +} + +// Verifies that once StopAutoClean is called, the goroutine actually exits and +// subsequent ticks are not observed. We can't directly observe goroutine exit +// but we can observe Clean not running: after stopping, a freshly expired +// entry must remain until the test manually cleans. +func TestAutoClean_StopHaltsCleaning(t *testing.T) { + var cleanCount int32 + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(time.Millisecond), + ) + // Poll until we see at least one clean happened by observing entry removal. + c.Set("a", 1) + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if _, _, s := c.TryGet("a"); s.IsMiss() { + atomic.StoreInt32(&cleanCount, 1) + break + } + time.Sleep(time.Millisecond) + } + if atomic.LoadInt32(&cleanCount) == 0 { + t.Fatal("AutoClean never observed cleaning before stop") + } + + c.StopAutoClean() + + // After stop, set an entry and verify it is NOT auto-cleaned within a + // window many multiples of the original interval. + c.Set("b", 2) + time.Sleep(20 * time.Millisecond) + // b is expired (MaxAge=1ms) but AutoClean is stopped, so TryGet still + // reports Miss only because of the expiry check — the key remains in + // the map. Verify Clean manually removes it. + c.Clean() + // A subsequent Get with a non-panicking miss should be called (i.e., the + // key is actually gone from the map, not just expired). + var missCalled bool + c.Get("b", func() (int, error) { + missCalled = true + return 0, nil + }) + if !missCalled { + t.Fatal("miss function should have run after manual Clean removed the entry") + } +} diff --git a/cache/cache.go b/cache/cache.go index f598979..1055884 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -282,6 +282,14 @@ func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeySta // We always return l.v and l.err. If this expires immediately, // get may return no value. We want to return at least the // value generated from this miss. + // + // If e.get above waited on a different loading (because a + // concurrent Swap/Set replaced e.p while we were racing), our + // l.v may still be unsynchronized with whoever called l.wg.Done + // — either the miss goroutine's setve or Swap's defer block, + // both of which write l.v before calling Done. Waiting on + // l.wg here provides the happens-before edge we need. + l.wg.Wait() return l.v, l.err, Miss } return v, err, Stale diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go new file mode 100644 index 0000000..475a798 --- /dev/null +++ b/cache/concurrency_test.go @@ -0,0 +1,730 @@ +package cache + +import ( + "errors" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestSwap_CancelsInFlightGet verifies that Swap overrides a currently-loading +// entry: the goroutine waiting in Get sees the Swap value, and the underlying +// miss function's return value is discarded. +// +// Covers cache.go Swap defer block for !was.finalized() (the cancel path). +func TestSwap_CancelsInFlightGet(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + var missReturned int32 + + var getV int + var getS KeyState + var getDone sync.WaitGroup + getDone.Add(1) + go func() { + defer getDone.Done() + v, _, s := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + atomic.StoreInt32(&missReturned, 1) + return 99, nil + }) + getV, getS = v, s + }() + + <-missStart // miss function has begun + + // Swap while the miss is still running. + old, _, oldS := c.Swap("k", 7) + // The entry was loading; there is no prior value to return, so Swap + // returns zero/Miss. + if oldS != Miss { + t.Fatalf("Swap old state: got %v want Miss", oldS) + } + _ = old + + // Let the miss function finish. Its value MUST be discarded. + close(missRelease) + getDone.Wait() + + // Get waited on the (now-finalized-by-Swap) loading and saw Swap's value. + if getV != 7 { + t.Fatalf("Get value = %d, want 7 (Swap should cancel miss)", getV) + } + // The state observed by the Get caller is Miss because from their + // perspective it was a miss (they supplied the miss fn). The value is + // what Swap finalized. + _ = getS + + // Verify the cache holds Swap's value, not the miss's. + v, _, s := c.TryGet("k") + if v != 7 || !s.IsHit() { + t.Fatalf("TryGet after Swap: v=%d s=%v, want 7 Hit", v, s) + } + if atomic.LoadInt32(&missReturned) != 1 { + t.Fatal("miss function never returned") + } +} + +// TestSwap_OverExpiredFinalizedWithStale exercises the defer branch that +// reads the prior entry's stale when the prior entry has expired but a valid +// stale is still in its window. +// +// Covers cache.go Swap defer lines for expired-with-stale (lines 451-457). +func TestSwap_OverExpiredFinalizedWithStale(t *testing.T) { + c := New[string, int]( + MaxAge(5*time.Millisecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(15 * time.Millisecond) + // Entry is expired but stale is still valid. + old, _, oldS := c.Swap("k", 2) + if oldS != Stale { + t.Fatalf("Swap state: got %v want Stale", oldS) + } + if old != 1 { + t.Fatalf("Swap old: got %d want 1", old) + } +} + +// TestSwap_OverErroredWithStale covers the defer branch where the prior +// entry is finalized but holds an error, and a valid stale exists. +func TestSwap_OverErroredWithStale(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour)) + c.Set("k", 1) + // Replace the value with an error result. We drive this via Get so the + // cached state becomes (err, stale=1). + c.Expire("k") + _, _, _ = c.Get("k", func() (int, error) { + return 0, errors.New("boom") + }) + // Now the entry has err set and stale=1. + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale { + t.Fatalf("Swap state: got %v want Stale", oldS) + } + if old != 1 || oldErr != nil { + t.Fatalf("Swap old: v=%d err=%v, want v=1 err=nil", old, oldErr) + } +} + +// TestDelete_DuringInFlightLoad verifies that deleting while a load is in +// flight: (a) the in-flight Get still returns the miss fn's result, and (b) +// a subsequent Get re-triggers the miss fn. +func TestDelete_DuringInFlightLoad(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Add(1) + go func() { + defer getDone.Done() + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 5, nil + }) + getV = v + }() + + <-missStart + c.Delete("k") + close(missRelease) + getDone.Wait() + + if getV != 5 { + t.Fatalf("in-flight Get got %d, want 5", getV) + } + + // Next Get must re-run the miss. + var called bool + v, _, _ := c.Get("k", func() (int, error) { + called = true + return 6, nil + }) + if !called { + t.Fatal("miss function should have run after Delete") + } + if v != 6 { + t.Fatalf("post-Delete Get got %d, want 6", v) + } +} + +// TestExpire_NoOpDuringInFlightLoad verifies the documented behavior: Expire +// on an in-flight load is a no-op; the load completes with its normal TTL. +func TestExpire_NoOpDuringInFlightLoad(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getDone sync.WaitGroup + getDone.Add(1) + go func() { + defer getDone.Done() + c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 11, nil + }) + }() + + <-missStart + c.Expire("k") // must be a no-op while loading + close(missRelease) + getDone.Wait() + + v, _, s := c.TryGet("k") + if v != 11 || !s.IsHit() { + t.Fatalf("after in-flight Expire: v=%d s=%v, want 11 Hit (Expire should not have affected the load)", v, s) + } +} + +// TestCollapsing_NoCaching runs many concurrent Gets against a cache with +// MaxAge(0) (no caching). The documented use case is collapsing simultaneous +// queries for the same key. +// +// The guarantee is weaker than it looks: with del0, the loading finalizes +// expired, so the *next* goroutine that acquires c.mu after the leader's +// setve sees an expired entry and triggers a fresh miss. Concurrent arrivals +// still collapse in batches, but "batch" here means "whatever queued on a +// given loading's WaitGroup before it was Done'd," which is scheduler- +// dependent. +// +// We assert: (1) miss runs at least once, (2) collapsing happens (fewer miss +// calls than workers), (3) every returned value corresponds to some miss +// invocation. +func TestCollapsing_NoCaching(t *testing.T) { + c := New[string, int](MaxAge(0)) + const workers = 100 + + var missStart sync.WaitGroup + missStart.Add(1) + missRelease := make(chan struct{}) + var missCalls int32 + + var wg sync.WaitGroup + wg.Add(workers) + vals := make([]int, workers) + for i := range workers { + go func(i int) { + defer wg.Done() + v, _, _ := c.Get("k", func() (int, error) { + n := atomic.AddInt32(&missCalls, 1) + if n == 1 { + missStart.Done() + <-missRelease + } + return int(n), nil + }) + vals[i] = v + }(i) + } + + missStart.Wait() + time.Sleep(5 * time.Millisecond) // let as many callers as possible queue + close(missRelease) + wg.Wait() + + calls := atomic.LoadInt32(&missCalls) + if calls < 1 { + t.Fatalf("miss never called") + } + if calls >= workers { + t.Fatalf("no collapsing observed: miss called %d times across %d Gets", calls, workers) + } + for i, v := range vals { + if v < 1 || int32(v) > calls { + t.Fatalf("goroutine %d got value %d, outside valid miss-call range [1,%d]", i, v, calls) + } + } + + // Nothing is cached. A fresh Get runs the miss again. + var called bool + c.Get("k", func() (int, error) { + called = true + return 99, nil + }) + if !called { + t.Fatal("after collapsed Get, cache should not have retained a value") + } +} + +// TestCleanUnderConcurrency stresses Clean racing with concurrent +// Set/Get/Delete to catch races in the clean path. Correctness here is that +// the test finishes without -race complaints or panics; we also assert that +// no value ever mysteriously appears that we never Set. +func TestCleanUnderConcurrency(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]( + MaxAge(time.Millisecond), + MaxStaleAge(time.Millisecond), + ) + + var stop atomic.Bool + var wg sync.WaitGroup + + // Setters. + for w := range 4 { + wg.Add(1) + go func(w int) { + defer wg.Done() + for !stop.Load() { + for i := range 64 { + c.Set(i, i*1000+w) + } + } + }(w) + } + + // Getters. + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range 64 { + v, _, s := c.TryGet(i) + if s.IsHit() { + // Value is i*1000+worker; must satisfy the invariant. + if v%1000 > 3 || v/1000 != i { + t.Errorf("impossible value v=%d for key=%d", v, i) + return + } + } + } + } + }() + } + + // Deleters. + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range 64 { + c.Delete(i) + } + } + }() + } + + // Cleaner. + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + c.Clean() + time.Sleep(200 * time.Microsecond) + } + }() + + time.Sleep(500 * time.Millisecond) + stop.Store(true) + wg.Wait() +} + +// TestSwap_OverPromotingDelete forces two rare branches simultaneously: +// +// 1. Swap's unlocked fast path observes an entry whose pointer is the +// promotingDelete sentinel (cache.go:474-475) and falls through to the +// locked path. +// 2. promote's inner CAS(nil, pd) fails because a concurrent Swap wrote a +// fresh loading into e.p between promote's Load and CAS (cache.go:636). +// +// Both windows are ~nanoseconds wide; we stress with many deleters + many +// swappers + an aggressive Range driver for long enough that both branches +// land. On a fast machine, either can land within a few hundred ms, but we +// leave a generous budget for slow CI runners. +func TestSwap_OverPromotingDelete(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + const readKeys = 32 + // Prime stable keys into the read map. + for i := range readKeys { + c.Set(i, i) + } + c.Range(func(int, int, error) bool { return true }) + + var wg sync.WaitGroup + var stop atomic.Bool + + // Deleters churn the primed keys, repeatedly creating the nil-e.p window + // that promote's CAS(nil, pd) targets. + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range readKeys { + c.Delete(i) + } + } + }() + } + // Swappers on the same primed keys exercise the unlocked fast-path CAS + // that races with promote. + for w := range 8 { + wg.Add(1) + go func(w int) { + defer wg.Done() + for !stop.Load() { + for i := range readKeys { + c.Swap(i, i*1000+w) + } + } + }(w) + } + // Churners add+delete NEW keys continuously so that dirty stays populated, + // which keeps r.incomplete=true and forces Range to call promote on every + // iteration. Without this, the initial promote fires once and subsequent + // Ranges skip the work. + for w := range 4 { + wg.Add(1) + go func(w int) { + defer wg.Done() + base := readKeys + w*10000 + for !stop.Load() { + for i := range 64 { + c.Set(base+i, i) + c.Delete(base + i) + } + } + }(w) + } + // Range callers hammer promote. + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + c.Range(func(int, int, error) bool { return true }) + } + }() + } + + time.Sleep(2 * time.Second) + stop.Store(true) + wg.Wait() +} + +// TestTryCAS_RetryOnRace exercises the tryCAS CAS-loop retry path by running +// many concurrent CompareAndSwap calls for the same key. When one thread's CAS +// succeeds, others retry (loading the new value) and may observe a value that +// no longer matches `old`, exiting the loop. +func TestTryCAS_RetryOnRace(t *testing.T) { + c := New[string, int]() + c.Set("k", 0) + + const workers = 32 + const iters = 1000 + + var wg sync.WaitGroup + var successes int32 + wg.Add(workers) + for range workers { + go func() { + defer wg.Done() + for range iters { + for { + v, _, s := c.TryGet("k") + if !s.IsHit() { + break + } + if c.CompareAndSwap("k", v, v+1) { + atomic.AddInt32(&successes, 1) + break + } + runtime.Gosched() + } + } + }() + } + wg.Wait() + + v, _, _ := c.TryGet("k") + if int32(v) != successes { + t.Fatalf("value %d != successful CAS count %d", v, successes) + } + if successes != workers*iters { + t.Fatalf("successes %d != expected %d (every worker should eventually succeed its increments)", successes, workers*iters) + } +} + +// TestCompareAndDelete_RaceOnDelete stresses the CompareAndDelete path with +// concurrent deletes/sets to exercise both the retry loop inside tryCAS and +// the value-mismatch path after a concurrent mutation. +func TestCompareAndDelete_RaceOnDelete(t *testing.T) { + c := New[int, int]() + const workers = 16 + var wg sync.WaitGroup + wg.Add(workers * 2) + + var setDone atomic.Int64 + for w := range workers { + go func(w int) { + defer wg.Done() + for i := range 1000 { + c.Set(i%8, w) + setDone.Add(1) + } + }(w) + } + for range workers { + go func() { + defer wg.Done() + for i := range 1000 { + v, _, s := c.TryGet(i % 8) + if s.IsHit() { + c.CompareAndDelete(i%8, v) + } + } + }() + } + wg.Wait() + // No assertion on final state; the race-detector + lack of panic is the + // signal. +} + +// TestSetve_RacesWithSwap runs many iterations where a slow miss function +// races against a Swap on the same key. The purpose is to exercise the +// double-check-under-mutex branch of loading.setve (miss grabs l.mu after +// Swap has already finalized l under the same mutex in its defer block). +// +// The branch is race-dependent; we run many iterations to land it. The test +// is correctness-only (value observed by all Gets is consistent). +func TestSetve_RacesWithSwap(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 200 { + c := New[string, int]() + missRelease := make(chan struct{}) + var wg sync.WaitGroup + var gotV int32 + wg.Add(1) + go func() { + defer wg.Done() + v, _, _ := c.Get("k", func() (int, error) { + <-missRelease + return 1, nil + }) + atomic.StoreInt32(&gotV, int32(v)) + }() + // Allow the miss goroutine to enter the function, then release it + // close in time with the Swap. + runtime.Gosched() + close(missRelease) + old, _, _ := c.Swap("k", 2) + _ = old + wg.Wait() + v := atomic.LoadInt32(&gotV) + if v != 1 && v != 2 { + t.Fatalf("iter %d: got %d, want 1 or 2", iter, v) + } + } +} + +// TestPromote_ConcurrentDeleteOfNilEntry forces promote to observe an entry +// whose pointer is nil but becomes non-nil between the Load and the +// CompareAndSwap(nil, pd) inside promote's retry loop. Covers the reload +// statement inside promote. +// +// This exercises cache.go:promote retry loop. Deterministic coverage of the +// exact reload line requires a goroutine interleaving that we cannot force +// from the outside; we stress via concurrent Delete/Set/Range to try to hit +// it. +func TestPromote_ConcurrentDeleteOfNilEntry(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + for i := range 128 { + c.Set(i, i) + } + + var wg sync.WaitGroup + var stop atomic.Bool + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range 128 { + c.Delete(i) + c.Set(i, i) + } + } + }() + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + c.Range(func(int, int, error) bool { return true }) + } + }() + time.Sleep(100 * time.Millisecond) + stop.Store(true) + wg.Wait() +} + +// TestClean_NoopWhenStalesArePermanent covers the MaxStaleAge<0 early return +// in Clean (the "stales kept forever" opt-in). +func TestClean_NoopWhenStalesArePermanent(t *testing.T) { + c := New[string, int](MaxAge(time.Nanosecond), MaxStaleAge(-1)) + c.Set("k", 1) + time.Sleep(time.Millisecond) // main entry is now expired, stale is forever + c.Clean() // must be a no-op + // The stale is still queryable. + _, _, s := c.TryGet("k") + if s != Stale { + t.Fatalf("after Clean with MaxStaleAge=-1: state=%v want Stale", s) + } +} + +// TestGet_UnlockedMissThenLockedHit targets Get's locked-path Hit (cache.go +// lines 233-236): unlocked e.get returned Miss, but by the time we acquire +// c.mu and re-check, a concurrent Swap has published a fresh value for the +// same key in the read map. +// +// Requires the entry to live in the read map (not dirty). We force that by +// priming + Range (promote), then Expire to make the unlocked fast-path Miss, +// then race a Swap against a Get. Hitting the exact timing window requires +// multiple iterations. +func TestGet_UnlockedMissThenLockedHit(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 5000 { + c := New[int, int](MaxAge(time.Hour)) + c.Set(1, 1) + c.Range(func(int, int, error) bool { return true }) + c.Expire(1) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + c.Swap(1, iter+100) + }() + go func() { + defer wg.Done() + v, _, s := c.Get(1, func() (int, error) { return -1, nil }) + if s.IsHit() && v != iter+100 { + t.Errorf("iter %d: Hit but v=%d, want %d", iter, v, iter+100) + } + }() + wg.Wait() + } +} + +// TestCompareAndSwap_LockPromotedEntry targets the CompareAndSwap locked +// branch where the entry was only observable via dirty on fast path but has +// since been promoted into the read map (cache.go lines 532-534). Race window. +func TestCompareAndSwap_LockPromotedEntry(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 5000 { + c := New[int, int]() + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + c.Set(iter, iter) + c.Range(func(int, int, error) bool { return true }) + }() + go func() { + defer wg.Done() + c.CompareAndSwap(iter, iter, iter+1) + }() + wg.Wait() + } +} + +// TestPromote_CASRetryOnConcurrentWrite targets promote's reload-after-failed- +// CAS line (cache.go line 636). The condition: promote holds c.mu, sees +// e.p=nil, tries CAS(nil, c.pd). The CAS fails because a concurrent goroutine +// (Get under c.mu could not run; Swap via unlocked fast path could not, since +// it requires e in read map — wait, it IS in read map here — so this window +// opens when Swap's unlocked CAS loop is active for an entry that was deleted +// during promote's iteration). +// +// We exercise by having many concurrent Swap/Delete/Range cycles so promote +// encounters entries mid-flux. Race-dependent; stress only. +func TestPromote_CASRetryOnConcurrentWrite(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + for i := range 64 { + c.Set(i, i) + } + c.Range(func(int, int, error) bool { return true }) + + var stop atomic.Bool + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range 64 { + c.Delete(i) + c.Swap(i, i*2) + } + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + c.Range(func(int, int, error) bool { return true }) + } + }() + time.Sleep(100 * time.Millisecond) + stop.Store(true) + wg.Wait() +} + +// TestMemoryBounded churns a fixed key set in and out of the cache. We don't +// assert exact bytes; we assert that after N rounds of (Set, Get, Delete, +// Range-triggered promote), the resident entry count is bounded by the number +// of live keys. This would catch a regression where dirty-map bookkeeping +// leaked entries. +func TestMemoryBounded(t *testing.T) { + c := New[int, int]() + const liveKeys = 32 + const rounds = 2000 + + for r := range rounds { + for i := range liveKeys { + c.Set(r*liveKeys+i, i) + } + // Delete the ones we just wrote except for `liveKeys` canonical keys. + for i := range liveKeys { + c.Delete(r*liveKeys + i) + } + for i := range liveKeys { + c.Set(i, i) + } + // Trigger promote. + c.Range(func(int, int, error) bool { return true }) + } + + count := 0 + c.Range(func(int, int, error) bool { count++; return true }) + if count > liveKeys { + t.Fatalf("after %d rounds: %d live entries, want <= %d", rounds, count, liveKeys) + } +} diff --git a/cache/wrappers_test.go b/cache/wrappers_test.go new file mode 100644 index 0000000..9cdada7 --- /dev/null +++ b/cache/wrappers_test.go @@ -0,0 +1,289 @@ +package cache + +import ( + "errors" + "sort" + "sync/atomic" + "testing" + "time" +) + +// Item delegates to Cache[struct{}, V]. These tests exercise every method so +// the thin wrappers contribute to coverage and so a refactor of the underlying +// Cache that forgets to propagate a method would be caught. + +func TestItem_BasicGetAndCaches(t *testing.T) { + i := NewItem[int]() + var calls int32 + v, err, s := i.Get(func() (int, error) { + atomic.AddInt32(&calls, 1) + return 42, nil + }) + if v != 42 || err != nil || !s.IsMiss() { + t.Fatalf("first Get: got v=%d err=%v s=%v, want v=42 err=nil Miss", v, err, s) + } + v, err, s = i.Get(func() (int, error) { + atomic.AddInt32(&calls, 1) + return -1, nil + }) + if v != 42 || err != nil || !s.IsHit() { + t.Fatalf("second Get: got v=%d err=%v s=%v, want v=42 err=nil Hit", v, err, s) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("miss fn called %d times, want 1", got) + } +} + +func TestItem_TryGet(t *testing.T) { + i := NewItem[string]() + _, _, s := i.TryGet() + if !s.IsMiss() { + t.Fatalf("TryGet on empty item: state=%v want Miss", s) + } + i.Set("hello") + v, err, s := i.TryGet() + if v != "hello" || err != nil || !s.IsHit() { + t.Fatalf("TryGet after Set: v=%q err=%v s=%v", v, err, s) + } +} + +func TestItem_Delete(t *testing.T) { + i := NewItem[int]() + i.Set(7) + v, err, s := i.Delete() + if v != 7 || err != nil || !s.IsHit() { + t.Fatalf("Delete of set value: v=%d err=%v s=%v", v, err, s) + } + _, _, s = i.TryGet() + if !s.IsMiss() { + t.Fatalf("TryGet after Delete: state=%v want Miss", s) + } + // Deleting an already-empty item is a no-op. + _, _, s = i.Delete() + if !s.IsMiss() { + t.Fatalf("Delete of empty item: state=%v want Miss", s) + } +} + +func TestItem_Expire(t *testing.T) { + i := NewItem[int](MaxAge(time.Hour)) + i.Set(99) + if _, _, s := i.TryGet(); !s.IsHit() { + t.Fatalf("before Expire: want Hit") + } + i.Expire() + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after Expire: want Miss") + } +} + +func TestItem_Swap(t *testing.T) { + i := NewItem[int]() + old, err, s := i.Swap(5) + if err != nil || !s.IsMiss() { + t.Fatalf("first Swap: old=%d err=%v s=%v, want Miss", old, err, s) + } + old, err, s = i.Swap(6) + if old != 5 || err != nil || !s.IsHit() { + t.Fatalf("second Swap: old=%d err=%v s=%v, want 5 Hit", old, err, s) + } + if v, _, _ := i.TryGet(); v != 6 { + t.Fatalf("after second Swap: TryGet=%d, want 6", v) + } +} + +func TestItem_CompareAndSwap(t *testing.T) { + i := NewItem[int]() + i.Set(10) + if !i.CompareAndSwap(10, 11) { + t.Fatal("CAS 10->11 should succeed") + } + if i.CompareAndSwap(10, 12) { + t.Fatal("CAS 10->12 should fail after value is 11") + } + if v, _, _ := i.TryGet(); v != 11 { + t.Fatalf("after CAS: TryGet=%d, want 11", v) + } +} + +func TestItem_CompareAndDelete(t *testing.T) { + i := NewItem[int]() + i.Set(7) + if i.CompareAndDelete(8) { + t.Fatal("CAD of wrong value should fail") + } + if !i.CompareAndDelete(7) { + t.Fatal("CAD of correct value should succeed") + } + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after CAD: state=%v want Miss", s) + } +} + +func TestItem_Clear(t *testing.T) { + i := NewItem[int]() + i.Set(1) + i.Clear() + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after Clear: state=%v want Miss", s) + } +} + +func TestItem_ZeroValue(t *testing.T) { + // Docstring says zero-value Item is valid. + var i Item[int] + v, err, s := i.Get(func() (int, error) { return 3, nil }) + if v != 3 || err != nil || !s.IsMiss() { + t.Fatalf("zero Item Get: v=%d err=%v s=%v", v, err, s) + } + if v, _, _ := i.TryGet(); v != 3 { + t.Fatalf("zero Item TryGet: v=%d want 3", v) + } +} + +// Set is a key-only cache over Cache[K, struct{}]. + +func TestSet_BasicGet(t *testing.T) { + s := NewSet[string]() + var calls int32 + err, ks := s.Get("a", func() error { + atomic.AddInt32(&calls, 1) + return nil + }) + if err != nil || !ks.IsMiss() { + t.Fatalf("first Get: err=%v state=%v want nil Miss", err, ks) + } + err, ks = s.Get("a", func() error { + atomic.AddInt32(&calls, 1) + return nil + }) + if err != nil || !ks.IsHit() { + t.Fatalf("second Get: err=%v state=%v want nil Hit", err, ks) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("miss fn called %d times, want 1", got) + } +} + +func TestSet_GetError(t *testing.T) { + s := NewSet[string]() + wantErr := errors.New("boom") + err, ks := s.Get("a", func() error { return wantErr }) + if !errors.Is(err, wantErr) || !ks.IsMiss() { + t.Fatalf("Get with error: err=%v state=%v", err, ks) + } +} + +func TestSet_TryGet(t *testing.T) { + s := NewSet[string]() + _, ks := s.TryGet("missing") + if !ks.IsMiss() { + t.Fatalf("TryGet missing: state=%v", ks) + } + s.Set("k") + err, ks := s.TryGet("k") + if err != nil || !ks.IsHit() { + t.Fatalf("TryGet after Set: err=%v state=%v", err, ks) + } +} + +func TestSet_Delete(t *testing.T) { + s := NewSet[string]() + s.Set("k") + _, ks := s.Delete("k") + if !ks.IsHit() { + t.Fatalf("Delete existing: state=%v want Hit", ks) + } + _, ks = s.Delete("k") + if !ks.IsMiss() { + t.Fatalf("Delete missing: state=%v want Miss", ks) + } +} + +func TestSet_Expire(t *testing.T) { + s := NewSet[string](MaxAge(time.Hour)) + s.Set("k") + s.Expire("k") + if _, ks := s.TryGet("k"); !ks.IsMiss() { + t.Fatalf("TryGet after Expire: state=%v want Miss", ks) + } +} + +func TestSet_Range(t *testing.T) { + s := NewSet[string]() + for _, k := range []string{"a", "b", "c"} { + s.Set(k) + } + var got []string + s.Range(func(k string, err error) bool { + if err != nil { + t.Errorf("unexpected err for key %q: %v", k, err) + } + got = append(got, k) + return true + }) + sort.Strings(got) + want := []string{"a", "b", "c"} + if len(got) != len(want) { + t.Fatalf("Range yielded %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("Range yielded %v, want %v", got, want) + } + } + + // Early stop. + count := 0 + s.Range(func(string, error) bool { + count++ + return false + }) + if count != 1 { + t.Fatalf("Range early-stop visited %d, want 1", count) + } +} + +func TestSet_Clean(t *testing.T) { + s := NewSet[string](MaxAge(time.Nanosecond)) + s.Set("k") + time.Sleep(time.Millisecond) + s.Clean() + if _, ks := s.TryGet("k"); !ks.IsMiss() { + t.Fatalf("TryGet after Clean on expired: state=%v want Miss", ks) + } +} + +func TestSet_Clear(t *testing.T) { + s := NewSet[string]() + s.Set("a") + s.Set("b") + s.Clear() + if _, ks := s.TryGet("a"); !ks.IsMiss() { + t.Fatalf("after Clear: state=%v want Miss", ks) + } + // Range yields nothing. + var got []string + s.Range(func(k string, _ error) bool { + got = append(got, k) + return true + }) + if len(got) != 0 { + t.Fatalf("Range after Clear: %v, want empty", got) + } +} + +func TestSet_StopAutoClean(t *testing.T) { + s := NewSet[string](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + s.StopAutoClean() + // Calling twice is safe (sync.Once on the underlying Cache). + s.StopAutoClean() +} + +func TestSet_ZeroValue(t *testing.T) { + var s Set[string] + err, ks := s.Get("a", func() error { return nil }) + if err != nil || !ks.IsMiss() { + t.Fatalf("zero Set Get: err=%v state=%v", err, ks) + } +} From 8e06ee51beb4c90c2b8705e417b7af08377f1111 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Tue, 21 Apr 2026 13:28:59 -0600 Subject: [PATCH 03/21] cache: add concurrent hash-trie in isolation Introduces cache/trie.go: a generic concurrent hash-trie map modeled on Go's internal/sync.HashTrieMap, but implemented entirely in userspace (hash/maphash for the runtime's typed hasher, unsafe.Sizeof(uintptr(0)) for pointer-size, no internal/abi or internal/goarch). The trie is not yet wired into Cache; that happens in a later commit. Design: - Root is a lazily-allocated indirect node, atomically swapped on Clear so readers holding the old root keep operating on it. - Interior nodes hold 16 atomic child pointers; leaves hold a key, an atomic value slot (*V), and an atomic overflow pointer for hash-collision chains. - Load is lock-free; LoadOrStore and Delete take the parent's per-bucket mutex. Delete prunes empty parents bottom-up while holding locks hand-over-hand. - expand splits a leaf into a subtree on hash-prefix collision; full hash collisions chain through overflow. Tests (cache/trie_test.go) cover: zero-value trie, basic store/load, load-or-store semantics with nil-value initialization, single and collision-chain delete (head and mid-chain), large-key stress (20k keys) through expand, full hash collisions via a test-only hashFn hook, concurrent disjoint keys, concurrent contention on the same key, concurrent delete-prune, range-during-mutation, and the deleteEntry lock-retry paths under sustained delete contention. Coverage is 99.1%. The five uncovered statements are defensive panics at invariant-violation points (tree deeper than the hash has bits, node type discriminator corrupted, etc.) that cannot fire without memory corruption; each is documented inline with the reason it is unreachable. Also: - Scale TestMaxIdleAge sleep windows up by ~4x so the timing-based assertions tolerate -count=3 -race load. The previous 30ms-within- 50ms-window spacing had no margin for scheduler delay. - Fix TestSwap_CancelsInFlightGet: synchronize on the miss goroutine actually completing (via a missDone channel) before asserting the miss ran. Previously the test asserted a flag set by a goroutine that hadn't been scheduled to run yet under heavy load. - Modernize remaining int-range for-loops. --- cache/cache_test.go | 53 ++-- cache/concurrency_test.go | 25 +- cache/trie.go | 429 +++++++++++++++++++++++++++++ cache/trie_test.go | 554 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 1020 insertions(+), 41 deletions(-) create mode 100644 cache/trie.go create mode 100644 cache/trie_test.go diff --git a/cache/cache_test.go b/cache/cache_test.go index d170261..4827063 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -58,7 +58,7 @@ func TestCollapsedGet(t *testing.T) { calls int64 ps = make(chan *int) ) - for i := 0; i < niter; i++ { + for range niter { go func() { p, _, _ := c.Get("foo", func() (*int, error) { if atomic.AddInt64(&calls, 1) != 1 { @@ -70,7 +70,7 @@ func TestCollapsedGet(t *testing.T) { ps <- p }() } - for i := 0; i < niter; i++ { + for range niter { p := <-ps if p != r { t.Error("pointer mismatch") @@ -118,7 +118,7 @@ func TestSetExpire(t *testing.T) { c.Delete(k) c.Set(k, 10) - for i := 0; i < 2; i++ { + for range 2 { var wg sync.WaitGroup wg.Add(10) go func() { defer wg.Done(); c.Range(func(string, int, error) bool { return false }) }() @@ -282,7 +282,7 @@ func TestExpires(t *testing.T) { gch := make(chan got[string], 10) ch := make(chan struct{}) - for i := 0; i < 10; i++ { + for range 10 { go func() { v, err, s := c.Get("foo", func() (string, error) { <-ch @@ -296,7 +296,7 @@ func TestExpires(t *testing.T) { // long, so not all Get's could be stacked at once. time.Sleep(100 * time.Millisecond) close(ch) - for i := 0; i < 10; i++ { + for range 10 { g := <-gch if g.s == Miss { vcheck(t, g, got[string]{"bar", nil, Miss}) @@ -365,22 +365,26 @@ func TestExpires(t *testing.T) { } func TestMaxIdleAge(t *testing.T) { + // Scale all time windows generously — these tests are timing-based and + // fail under heavy -race/stress load if the sleep/access margin is thin. + // MaxIdleAge alone: entry stays alive while accessed within the window, // expires after inactivity. t.Run("alone", func(t *testing.T) { - c := New[string, string](MaxIdleAge(50 * time.Millisecond)) + const idle = 200 * time.Millisecond + c := New[string, string](MaxIdleAge(idle)) c.Get("foo", func() (string, error) { return "bar", nil }) - time.Sleep(10 * time.Millisecond) + time.Sleep(40 * time.Millisecond) // Access within the idle window — should still be a hit and extend. - for i := 0; i < 5; i++ { + for range 5 { v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) - time.Sleep(30 * time.Millisecond) // each access resets the 50ms idle + time.Sleep(40 * time.Millisecond) // ≪ idle, plenty of margin } - // Now stop accessing. After 50ms of inactivity, it should expire. - time.Sleep(60 * time.Millisecond) + // Now stop accessing. After idle of inactivity, it should expire. + time.Sleep(idle + 50*time.Millisecond) v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) @@ -388,51 +392,54 @@ func TestMaxIdleAge(t *testing.T) { // MaxIdleAge + MaxAge: initial expiry uses MaxAge, subsequent accesses // extend using MaxIdleAge. t.Run("with_max_age", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + const ttl = 200 * time.Millisecond + c := New[string, string](MaxAge(ttl), MaxIdleAge(ttl)) c.Get("foo", func() (string, error) { return "bar", nil }) // Keep alive well past the original MaxAge. - for i := 0; i < 5; i++ { - time.Sleep(30 * time.Millisecond) + for range 5 { + time.Sleep(40 * time.Millisecond) v, err, s := c.Get("foo", func() (string, error) { return "baz", nil }) vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) } // Stop accessing, should expire. - time.Sleep(60 * time.Millisecond) + time.Sleep(ttl + 50*time.Millisecond) v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) // MaxIdleAge doesn't extend errors. t.Run("no_extend_errors", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + const ttl = 200 * time.Millisecond + c := New[string, string](MaxAge(ttl), MaxIdleAge(ttl)) c.Get("foo", func() (string, error) { return "", errors.New("err") }) - time.Sleep(10 * time.Millisecond) + time.Sleep(40 * time.Millisecond) // Access the errored entry — it should not be extended. v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", errors.New("err"), Hit}) // Wait past expiry — should be gone. - time.Sleep(50 * time.Millisecond) + time.Sleep(ttl + 50*time.Millisecond) v, err, s = c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) // Range doesn't extend. t.Run("range_no_extend", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + const ttl = 200 * time.Millisecond + c := New[string, string](MaxAge(ttl), MaxIdleAge(ttl)) c.Get("foo", func() (string, error) { return "bar", nil }) // Range over entries repeatedly — should NOT extend expiry. - for i := 0; i < 3; i++ { - time.Sleep(20 * time.Millisecond) + for range 3 { + time.Sleep(40 * time.Millisecond) c.Range(func(string, string, error) bool { return true }) } - // Should be expired by now (>50ms since creation, range didn't extend). - time.Sleep(20 * time.Millisecond) + // Should be expired by now (>ttl since creation, range didn't extend). + time.Sleep(ttl + 50*time.Millisecond) v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index 475a798..e167dc4 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -18,55 +18,44 @@ func TestSwap_CancelsInFlightGet(t *testing.T) { c := New[string, int]() missStart := make(chan struct{}) missRelease := make(chan struct{}) - var missReturned int32 + missDone := make(chan struct{}) var getV int - var getS KeyState var getDone sync.WaitGroup getDone.Add(1) go func() { defer getDone.Done() - v, _, s := c.Get("k", func() (int, error) { + v, _, _ := c.Get("k", func() (int, error) { close(missStart) <-missRelease - atomic.StoreInt32(&missReturned, 1) + close(missDone) return 99, nil }) - getV, getS = v, s + getV = v }() <-missStart // miss function has begun // Swap while the miss is still running. - old, _, oldS := c.Swap("k", 7) - // The entry was loading; there is no prior value to return, so Swap - // returns zero/Miss. + _, _, oldS := c.Swap("k", 7) if oldS != Miss { t.Fatalf("Swap old state: got %v want Miss", oldS) } - _ = old // Let the miss function finish. Its value MUST be discarded. close(missRelease) - getDone.Wait() + <-missDone // ensure the miss goroutine actually ran to completion + getDone.Wait() // ensure the Get goroutine returned - // Get waited on the (now-finalized-by-Swap) loading and saw Swap's value. if getV != 7 { t.Fatalf("Get value = %d, want 7 (Swap should cancel miss)", getV) } - // The state observed by the Get caller is Miss because from their - // perspective it was a miss (they supplied the miss fn). The value is - // what Swap finalized. - _ = getS // Verify the cache holds Swap's value, not the miss's. v, _, s := c.TryGet("k") if v != 7 || !s.IsHit() { t.Fatalf("TryGet after Swap: v=%d s=%v, want 7 Hit", v, s) } - if atomic.LoadInt32(&missReturned) != 1 { - t.Fatal("miss function never returned") - } } // TestSwap_OverExpiredFinalizedWithStale exercises the defer branch that diff --git a/cache/trie.go b/cache/trie.go new file mode 100644 index 0000000..80f4bef --- /dev/null +++ b/cache/trie.go @@ -0,0 +1,429 @@ +package cache + +import ( + "hash/maphash" + "sync" + "sync/atomic" + "unsafe" +) + +// trie is a concurrent hash-trie map, modeled on Go's internal/sync +// HashTrieMap. It is intended to replace Cache's read/dirty backing store. +// For now it is standalone and only exercised by trie_test.go. +// +// Design notes: +// +// - The root is a lazily-allocated indirect node, atomically swapped on +// Clear so readers that already loaded the root pointer keep operating +// on the old tree. +// +// - Interior nodes ("indirect") hold trieBranching atomic child pointers. +// Leaf nodes ("entry") hold a key, an atomic value slot (*V), and an +// atomic overflow pointer for hash-collision chains. +// +// - Load is lock-free: walk the tree with atomic loads, follow overflow +// on hash collision. +// +// - LoadOrStore and Delete take the *parent* indirect node's mutex +// (per-bucket locking) during mutation. After Delete, empty parents +// are pruned bottom-up while locks are held. +// +// - Hash and pointer-size primitives come from userspace: hash/maphash +// for the runtime's typed hasher, unsafe.Sizeof(uintptr(0)) for the +// hash bit budget. No internal/abi, no internal/goarch. + +const ( + trieBranchingLog2 = 4 + trieBranching = 1 << trieBranchingLog2 + trieBranchingMask = trieBranching - 1 +) + +// ptrBits is the number of hash bits consumed per trie walk. Using uintptr +// gives one hasher call on both 32-bit (32 bits consumed) and 64-bit (64 +// bits consumed) platforms; on 32-bit, maphash.Comparable internally calls +// the runtime hasher twice to synthesize 64 bits, of which we keep the low +// uintptr — a cheap extra hash call on rare hardware. +const ptrBits = uint(unsafe.Sizeof(uintptr(0))) * 8 + +// trieNode is the common header for both leaf (entry) and interior +// (indirect) nodes. The layout is carefully matched so a *trieNode can be +// unsafe-cast to either concrete type via the isEntry discriminator. +type trieNode[K comparable, V any] struct { + isEntry bool +} + +// trieIndirect is an interior node with trieBranching atomic child slots. +type trieIndirect[K comparable, V any] struct { + trieNode[K, V] + dead atomic.Bool + mu sync.Mutex + parent *trieIndirect[K, V] + children [trieBranching]atomic.Pointer[trieNode[K, V]] +} + +// trieEntry is a leaf node. The value slot p is an atomic pointer; a nil +// value slot means the entry is tombstoned but still wired into the trie +// (callers can atomically replace nil with a fresh value). The overflow +// chain handles hash-equal keys (full hash collisions). +type trieEntry[K comparable, V any] struct { + trieNode[K, V] + overflow atomic.Pointer[trieEntry[K, V]] + key K + p atomic.Pointer[V] +} + +func newTrieIndirect[K comparable, V any](parent *trieIndirect[K, V]) *trieIndirect[K, V] { + n := &trieIndirect[K, V]{parent: parent} + n.isEntry = false + return n +} + +func newTrieEntry[K comparable, V any](key K, value *V) *trieEntry[K, V] { + e := &trieEntry[K, V]{key: key} + e.isEntry = true + if value != nil { + e.p.Store(value) + } + return e +} + +// entry unsafe-casts a trieNode to its leaf form. Every call site checks +// n.isEntry first, so no runtime guard is needed here. +func (n *trieNode[K, V]) entry() *trieEntry[K, V] { + return (*trieEntry[K, V])(unsafe.Pointer(n)) +} + +// indirect unsafe-casts a trieNode to its interior form. Every call site +// checks !n.isEntry first, so no runtime guard is needed here. +func (n *trieNode[K, V]) indirect() *trieIndirect[K, V] { + return (*trieIndirect[K, V])(unsafe.Pointer(n)) +} + +// empty reports whether all child slots of i are nil. +func (i *trieIndirect[K, V]) empty() bool { + for j := range i.children { + if i.children[j].Load() != nil { + return false + } + } + return true +} + +type trie[K comparable, V any] struct { + inited atomic.Bool + initMu sync.Mutex + root atomic.Pointer[trieIndirect[K, V]] + seed maphash.Seed + + // hashFn, if non-nil, overrides maphash.Comparable for tests that need + // to construct hash collisions deterministically. Never set in + // production code. + hashFn func(K) uintptr +} + +func (t *trie[K, V]) init() { + if t.inited.Load() { + return + } + t.initMu.Lock() + defer t.initMu.Unlock() + if t.inited.Load() { + return + } + if t.hashFn == nil { + t.seed = maphash.MakeSeed() + } + t.root.Store(newTrieIndirect[K, V](nil)) + t.inited.Store(true) +} + +func (t *trie[K, V]) hash(k K) uintptr { + if t.hashFn != nil { + return t.hashFn(k) + } + return uintptr(maphash.Comparable(t.seed, k)) +} + +// loadEntry returns the entry for k, or nil if the key is not present. The +// lookup is lock-free: atomic loads walk the tree, then a final walk over +// the overflow chain checks for an exact key match. +func (t *trie[K, V]) loadEntry(k K) *trieEntry[K, V] { + if !t.inited.Load() { + return nil + } + h := t.hash(k) + i := t.root.Load() + shift := ptrBits + for shift != 0 { + shift -= trieBranchingLog2 + n := i.children[(h>>shift)&trieBranchingMask].Load() + if n == nil { + return nil + } + if n.isEntry { + for e := n.entry(); e != nil; e = e.overflow.Load() { + if e.key == k { + return e + } + } + return nil + } + i = n.indirect() + } + // Unreachable: the tree depth is bounded by ptrBits / trieBranchingLog2 + // (16 levels on 64-bit, 8 on 32-bit), so the loop always returns before + // shift reaches 0. Kept as a defensive assertion in case the tree is + // ever corrupted. + panic("cache: trie ran out of hash bits in loadEntry") +} + +// loadOrStoreEntry returns the existing entry for k if present. Otherwise it +// inserts a new entry initialized with value (which may be nil, meaning the +// caller will fill the value slot later) and returns it. The second return +// is true if an existing entry was loaded, false if a new one was stored. +// +// The insertion path takes the parent indirect node's lock. On hash prefix +// collision (two distinct keys sharing the same hash bits at the current +// depth) we build out indirection nodes until the bits diverge, via expand. +func (t *trie[K, V]) loadOrStoreEntry(k K, value *V) (*trieEntry[K, V], bool) { + t.init() + h := t.hash(k) + var i *trieIndirect[K, V] + var shift uint + var slot *atomic.Pointer[trieNode[K, V]] + var n *trieNode[K, V] + for { + i = t.root.Load() + shift = ptrBits + haveInsertPoint := false + for shift != 0 { + shift -= trieBranchingLog2 + slot = &i.children[(h>>shift)&trieBranchingMask] + n = slot.Load() + if n == nil { + haveInsertPoint = true + break + } + if n.isEntry { + for e := n.entry(); e != nil; e = e.overflow.Load() { + if e.key == k { + return e, true + } + } + haveInsertPoint = true + break + } + i = n.indirect() + } + if !haveInsertPoint { + // Unreachable: see the note in loadEntry above. + panic("cache: trie ran out of hash bits in loadOrStoreEntry") + } + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + break + } + i.mu.Unlock() + } + defer i.mu.Unlock() + + var oldEntry *trieEntry[K, V] + if n != nil { + oldEntry = n.entry() + for e := oldEntry; e != nil; e = e.overflow.Load() { + if e.key == k { + return e, true + } + } + } + + newEntry := newTrieEntry(k, value) + if oldEntry == nil { + slot.Store(&newEntry.trieNode) + } else { + slot.Store(t.expand(oldEntry, newEntry, h, shift, i)) + } + return newEntry, false +} + +// expand resolves an insert where a leaf already sits at the target slot +// but holds a different key. If the old and new keys hash identically, we +// chain the old entry onto the new entry's overflow list. Otherwise we +// build out indirect nodes until the next differing hash bit, then place +// both entries in distinct children. +func (t *trie[K, V]) expand(oldEntry, newEntry *trieEntry[K, V], newHash uintptr, shift uint, parent *trieIndirect[K, V]) *trieNode[K, V] { + oldHash := t.hash(oldEntry.key) + if oldHash == newHash { + newEntry.overflow.Store(oldEntry) + return &newEntry.trieNode + } + top := newTrieIndirect(parent) + cur := top + for { + if shift == 0 { + // Unreachable: oldHash and newHash differ (same-hash case + // returned above via the overflow chain), so some bit in + // [0, shift) must differ and the loop exits before shift=0. + panic("cache: trie ran out of hash bits in expand") + } + shift -= trieBranchingLog2 + oi := (oldHash >> shift) & trieBranchingMask + ni := (newHash >> shift) & trieBranchingMask + if oi != ni { + cur.children[oi].Store(&oldEntry.trieNode) + cur.children[ni].Store(&newEntry.trieNode) + break + } + next := newTrieIndirect(cur) + cur.children[oi].Store(&next.trieNode) + cur = next + } + return &top.trieNode +} + +// deleteEntry removes the entry for k from the trie. Returns the removed +// entry (or nil). Empty interior ancestors are pruned bottom-up. +func (t *trie[K, V]) deleteEntry(k K) *trieEntry[K, V] { + if !t.inited.Load() { + return nil + } + h := t.hash(k) + + // Walk to the entry, then lock the parent indirect node. We retry if we + // see the parent has been marked dead by concurrent pruning. + var i *trieIndirect[K, V] + var shift uint + var slot *atomic.Pointer[trieNode[K, V]] + var n *trieNode[K, V] + for { + i = t.root.Load() + shift = ptrBits + found := false + for shift != 0 { + shift -= trieBranchingLog2 + slot = &i.children[(h>>shift)&trieBranchingMask] + n = slot.Load() + if n == nil { + return nil + } + if n.isEntry { + found = true + break + } + i = n.indirect() + } + if !found { + // Unreachable: see the note in loadEntry above. + panic("cache: trie ran out of hash bits in deleteEntry") + } + i.mu.Lock() + if i.dead.Load() { + i.mu.Unlock() + continue + } + n = slot.Load() + if n == nil || !n.isEntry { + i.mu.Unlock() + return nil + } + break + } + + // Under i.mu, remove k from the overflow chain. If k was the head and + // had no overflow, slot becomes nil; otherwise slot gets the new head. + head := n.entry() + removed, newHead, found := trieRemoveFromChain(head, k) + if !found { + i.mu.Unlock() + return nil + } + if newHead != nil { + slot.Store(&newHead.trieNode) + i.mu.Unlock() + return removed + } + slot.Store(nil) + + // Prune empty parents. Walk up while holding locks as a hand-over-hand + // from child to parent. + for i.parent != nil && i.empty() { + if shift == ptrBits { + // Unreachable: pruning walks strictly upward from a leaf, so + // shift is always < ptrBits when we enter (we consumed at + // least trieBranchingLog2 bits on descent). Kept as a + // defensive assertion. + panic("cache: trie shift overflow in deleteEntry pruning") + } + shift += trieBranchingLog2 + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(h>>shift)&trieBranchingMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return removed +} + +// trieRemoveFromChain walks the overflow chain starting at head, removes +// the entry whose key equals k, and returns the removed entry, the new +// chain head (which may be nil if the only entry was the head), and +// whether an entry was actually removed. +func trieRemoveFromChain[K comparable, V any](head *trieEntry[K, V], k K) (removed, newHead *trieEntry[K, V], found bool) { + if head.key == k { + return head, head.overflow.Load(), true + } + prev := head + for { + next := prev.overflow.Load() + if next == nil { + return nil, head, false + } + if next.key == k { + prev.overflow.Store(next.overflow.Load()) + return next, head, true + } + prev = next + } +} + +// walk iterates every entry in the trie, calling f for each. If f returns +// false, iteration stops. No snapshot is taken; concurrent mutations may +// or may not be observed. A given key is never visited more than once. +func (t *trie[K, V]) walk(f func(*trieEntry[K, V]) bool) { + if !t.inited.Load() { + return + } + trieWalkIndirect(t.root.Load(), f) +} + +func trieWalkIndirect[K comparable, V any](i *trieIndirect[K, V], f func(*trieEntry[K, V]) bool) bool { + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + if !n.isEntry { + if !trieWalkIndirect(n.indirect(), f) { + return false + } + continue + } + for e := n.entry(); e != nil; e = e.overflow.Load() { + if !f(e) { + return false + } + } + } + return true +} + +// clear replaces the root with a fresh empty indirect node. Concurrent +// readers that already loaded the old root keep walking it; the old tree +// becomes unreachable once all such readers finish. +func (t *trie[K, V]) clear() { + t.init() + t.root.Store(newTrieIndirect[K, V](nil)) +} diff --git a/cache/trie_test.go b/cache/trie_test.go new file mode 100644 index 0000000..cf191d1 --- /dev/null +++ b/cache/trie_test.go @@ -0,0 +1,554 @@ +package cache + +import ( + "fmt" + "math/rand/v2" + "sync" + "sync/atomic" + "testing" +) + +// Helper: store or replace the value slot with a fresh pointer carrying v. +// The trie owns the entry; the caller owns the value pointer identity. +func triePut[K comparable, V any](t *trie[K, V], k K, v V) *V { + p := &v + e, _ := t.loadOrStoreEntry(k, p) + e.p.Store(p) + return p +} + +func trieGet[K comparable, V any](t *trie[K, V], k K) (V, bool) { + e := t.loadEntry(k) + if e == nil { + return *new(V), false + } + p := e.p.Load() + if p == nil { + return *new(V), false + } + return *p, true +} + +func TestTrie_ZeroValueLoad(t *testing.T) { + var tr trie[string, int] + if e := tr.loadEntry("missing"); e != nil { + t.Fatalf("loadEntry on zero-value trie returned %v, want nil", e) + } + var visited int + tr.walk(func(*trieEntry[string, int]) bool { visited++; return true }) + if visited != 0 { + t.Fatalf("walk on zero-value trie visited %d, want 0", visited) + } + if e := tr.deleteEntry("missing"); e != nil { + t.Fatalf("deleteEntry on zero-value trie returned %v, want nil", e) + } +} + +func TestTrie_StoreLoad(t *testing.T) { + var tr trie[string, int] + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + if v, ok := trieGet(&tr, "a"); !ok || v != 1 { + t.Fatalf("get a: v=%d ok=%v, want 1 true", v, ok) + } + if v, ok := trieGet(&tr, "b"); !ok || v != 2 { + t.Fatalf("get b: v=%d ok=%v, want 2 true", v, ok) + } + if _, ok := trieGet(&tr, "c"); ok { + t.Fatal("get c should be missing") + } +} + +func TestTrie_LoadOrStoreExisting(t *testing.T) { + var tr trie[string, int] + v1 := 1 + e1, loaded := tr.loadOrStoreEntry("k", &v1) + if loaded { + t.Fatal("first loadOrStoreEntry should not report loaded") + } + v2 := 2 + e2, loaded := tr.loadOrStoreEntry("k", &v2) + if !loaded { + t.Fatal("second loadOrStoreEntry should report loaded") + } + if e1 != e2 { + t.Fatal("loadOrStoreEntry should return the same entry object on subsequent calls") + } + // The stored value is v1's pointer; v2 was ignored. + if got := *e2.p.Load(); got != 1 { + t.Fatalf("stored value = %d, want 1 (new value must not overwrite)", got) + } +} + +func TestTrie_LoadOrStoreWithNilValue(t *testing.T) { + // Cache's Get path constructs an entry with no value yet, then fills it. + // Verify loadOrStoreEntry(k, nil) is legal and the entry.p is nil. + var tr trie[string, int] + e, loaded := tr.loadOrStoreEntry("k", nil) + if loaded { + t.Fatal("expected not-loaded for first insert") + } + if e.p.Load() != nil { + t.Fatal("nil-initialized entry should have nil value slot") + } + v := 42 + e.p.Store(&v) + if got, ok := trieGet(&tr, "k"); !ok || got != 42 { + t.Fatalf("after Store: got=%d ok=%v, want 42 true", got, ok) + } +} + +func TestTrie_Delete(t *testing.T) { + var tr trie[string, int] + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + + e := tr.deleteEntry("a") + if e == nil || e.key != "a" { + t.Fatalf("deleteEntry(a) = %v, want entry for a", e) + } + if _, ok := trieGet(&tr, "a"); ok { + t.Fatal("after delete, a should be missing") + } + if _, ok := trieGet(&tr, "b"); !ok { + t.Fatal("after delete of a, b should still be present") + } + if e := tr.deleteEntry("a"); e != nil { + t.Fatalf("second deleteEntry(a) = %v, want nil", e) + } +} + +func TestTrie_DeletePrunesEmptyParents(t *testing.T) { + // Insert many keys so the tree grows, then delete them all. After, the + // root should be empty (all children nil). + var tr trie[int, int] + const n = 1000 + for i := range n { + triePut(&tr, i, i) + } + for i := range n { + if e := tr.deleteEntry(i); e == nil { + t.Fatalf("deleteEntry(%d) unexpectedly returned nil", i) + } + } + root := tr.root.Load() + for i := range root.children { + if n := root.children[i].Load(); n != nil { + t.Fatalf("root.children[%d] = %v, want nil (pruning should have cleared)", i, n) + } + } +} + +func TestTrie_Walk(t *testing.T) { + var tr trie[int, int] + const n = 256 + for i := range n { + triePut(&tr, i, i*10) + } + seen := make(map[int]int) + tr.walk(func(e *trieEntry[int, int]) bool { + if p := e.p.Load(); p != nil { + seen[e.key] = *p + } + return true + }) + if len(seen) != n { + t.Fatalf("walk saw %d entries, want %d", len(seen), n) + } + for k, v := range seen { + if v != k*10 { + t.Fatalf("seen[%d] = %d, want %d", k, v, k*10) + } + } +} + +func TestTrie_WalkEarlyStop(t *testing.T) { + var tr trie[int, int] + for i := range 100 { + triePut(&tr, i, i) + } + var count int + tr.walk(func(*trieEntry[int, int]) bool { + count++ + return count < 5 + }) + if count != 5 { + t.Fatalf("walk visited %d, want 5 (early stop)", count) + } +} + +func TestTrie_Clear(t *testing.T) { + var tr trie[int, int] + for i := range 100 { + triePut(&tr, i, i) + } + tr.clear() + for i := range 100 { + if _, ok := trieGet(&tr, i); ok { + t.Fatalf("after clear, key %d should be missing", i) + } + } + var count int + tr.walk(func(*trieEntry[int, int]) bool { count++; return true }) + if count != 0 { + t.Fatalf("walk after clear visited %d, want 0", count) + } + // Trie is still usable after clear. + triePut(&tr, 1, 1) + if v, ok := trieGet(&tr, 1); !ok || v != 1 { + t.Fatalf("reuse after clear: v=%d ok=%v", v, ok) + } +} + +// TestTrie_ManyKeys exercises the expand path repeatedly: random keys will +// routinely collide on short hash prefixes, forcing the trie to extend its +// depth. 20k keys is enough that most hash prefixes of length 4 will +// collide multiple times. +func TestTrie_ManyKeys(t *testing.T) { + var tr trie[int, int] + const n = 20_000 + keys := rand.Perm(n) + for _, k := range keys { + triePut(&tr, k, k*2) + } + for _, k := range keys { + v, ok := trieGet(&tr, k) + if !ok || v != k*2 { + t.Fatalf("lookup(%d): v=%d ok=%v, want %d true", k, v, ok, k*2) + } + } + // Delete half and re-verify. + for i, k := range keys { + if i%2 == 0 { + tr.deleteEntry(k) + } + } + for i, k := range keys { + v, ok := trieGet(&tr, k) + switch i % 2 { + case 0: + if ok { + t.Fatalf("deleted key %d still present: v=%d", k, v) + } + case 1: + if !ok || v != k*2 { + t.Fatalf("surviving key %d: v=%d ok=%v, want %d true", k, v, ok, k*2) + } + } + } +} + +// TestTrie_FullHashCollision verifies the overflow-chain behavior by forcing +// two distinct keys to hash identically via the test-only hashFn hook. +func TestTrie_FullHashCollision(t *testing.T) { + var tr trie[string, int] + // Collide everything in a single hash bucket. + tr.hashFn = func(string) uintptr { return 0xdeadbeef } + + const n = 10 + for i := range n { + k := fmt.Sprintf("k%d", i) + triePut(&tr, k, i) + } + // All n entries share one overflow chain under the same leaf slot. + for i := range n { + k := fmt.Sprintf("k%d", i) + v, ok := trieGet(&tr, k) + if !ok || v != i { + t.Fatalf("get %q: v=%d ok=%v, want %d true", k, v, ok, i) + } + } + // Delete one from the middle; the rest remain. + tr.deleteEntry("k5") + if _, ok := trieGet(&tr, "k5"); ok { + t.Fatal("k5 should be deleted") + } + for i := range n { + if i == 5 { + continue + } + k := fmt.Sprintf("k%d", i) + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%q should survive delete of k5", k) + } + } +} + +// TestTrie_FullHashCollisionDeleteHead deletes the head of the overflow +// chain. The chain head is what the slot points at, so slot must be +// updated to the next entry. +func TestTrie_FullHashCollisionDeleteHead(t *testing.T) { + var tr trie[string, int] + tr.hashFn = func(string) uintptr { return 0x42 } + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + triePut(&tr, "c", 3) + + // Delete the most-recently-inserted (head of chain, per expand logic). + if e := tr.deleteEntry("c"); e == nil || e.key != "c" { + t.Fatalf("delete c: %v", e) + } + for _, k := range []string{"a", "b"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%s missing after delete of chain head", k) + } + } +} + +func TestTrie_LoadOrStoreDuringCollision(t *testing.T) { + // After a collision chain exists, loadOrStoreEntry for an already-present + // key must find it via the overflow walk, not append a duplicate. + var tr trie[string, int] + tr.hashFn = func(string) uintptr { return 0xabcd } + triePut(&tr, "x", 1) + triePut(&tr, "y", 2) + v := 999 + e, loaded := tr.loadOrStoreEntry("x", &v) + if !loaded { + t.Fatal("second loadOrStoreEntry(x) should report loaded") + } + if got := *e.p.Load(); got != 1 { + t.Fatalf("existing entry value=%d, want 1 (new value must be ignored)", got) + } +} + +// TestTrie_ConcurrentStoreLoad runs many goroutines performing +// loadOrStoreEntry and loadEntry over disjoint key ranges, verifying no +// race and eventual consistency. +func TestTrie_ConcurrentStoreLoad(t *testing.T) { + var tr trie[int, int] + const workers = 8 + const perWorker = 2000 + + var wg sync.WaitGroup + wg.Add(workers) + for w := range workers { + go func(w int) { + defer wg.Done() + base := w * perWorker + for i := range perWorker { + triePut(&tr, base+i, base+i) + } + for i := range perWorker { + v, ok := trieGet(&tr, base+i) + if !ok || v != base+i { + t.Errorf("worker %d, key %d: v=%d ok=%v", w, base+i, v, ok) + return + } + } + }(w) + } + wg.Wait() +} + +// TestTrie_ConcurrentSameKey hammers the same key with many goroutines +// doing loadOrStoreEntry. Exactly one must report loaded=false; all others +// must observe the same entry object. +func TestTrie_ConcurrentSameKey(t *testing.T) { + const workers = 64 + for iter := range 100 { + var tr trie[int, int] + var wg sync.WaitGroup + wg.Add(workers) + entries := make([]*trieEntry[int, int], workers) + loadeds := make([]bool, workers) + for w := range workers { + go func(w int) { + defer wg.Done() + v := w + e, loaded := tr.loadOrStoreEntry(iter, &v) + entries[w] = e + loadeds[w] = loaded + }(w) + } + wg.Wait() + + var inserted int + for _, l := range loadeds { + if !l { + inserted++ + } + } + if inserted != 1 { + t.Fatalf("iter %d: %d inserters, want exactly 1", iter, inserted) + } + first := entries[0] + for i, e := range entries { + if e != first { + t.Fatalf("iter %d: worker %d saw entry %p, want %p", iter, i, e, first) + } + } + } +} + +// TestTrie_ConcurrentDeletePrune exercises concurrent deletes across many +// keys so parent pruning fires from many angles simultaneously. +func TestTrie_ConcurrentDeletePrune(t *testing.T) { + var tr trie[int, int] + const n = 10_000 + for i := range n { + triePut(&tr, i, i) + } + const workers = 8 + var wg sync.WaitGroup + wg.Add(workers) + for w := range workers { + go func(w int) { + defer wg.Done() + for i := w; i < n; i += workers { + tr.deleteEntry(i) + } + }(w) + } + wg.Wait() + for i := range n { + if _, ok := trieGet(&tr, i); ok { + t.Fatalf("key %d should be deleted", i) + } + } + // After all deletions, root should be empty. + if root := tr.root.Load(); !root.empty() { + t.Fatal("root not pruned: has surviving children") + } +} + +// TestTrie_RangeDuringMutation runs walk concurrently with store and +// delete. No snapshot is promised; we assert only no-crash + that every +// observed key-value pair is plausible (key was stored, value is key*2 or +// nil). +func TestTrie_RangeDuringMutation(t *testing.T) { + var tr trie[int, int] + const n = 2000 + for i := range n { + triePut(&tr, i, i*2) + } + + var stop atomic.Bool + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + i := rand.IntN(n) + triePut(&tr, i, i*2) + tr.deleteEntry(i) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + tr.walk(func(e *trieEntry[int, int]) bool { + p := e.p.Load() + if p != nil && *p != e.key*2 { + t.Errorf("invariant break: key=%d value=%d", e.key, *p) + return false + } + return true + }) + } + }() + + // Let the race run briefly. + for range 50_000 { + i := rand.IntN(n) + if _, ok := trieGet(&tr, i); ok { + // just a read + } + } + stop.Store(true) + wg.Wait() +} + +// TestTrie_DeleteMissingInCollisionChain walks an overflow chain in search +// of a key that is never present, exercising the "walk to end without +// finding" branch of trieRemoveFromChain. +func TestTrie_DeleteMissingInCollisionChain(t *testing.T) { + var tr trie[string, int] + tr.hashFn = func(string) uintptr { return 0x1234 } + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + triePut(&tr, "c", 3) + // "missing" collides with a/b/c but was never stored. + if e := tr.deleteEntry("missing"); e != nil { + t.Fatalf("deleteEntry of never-stored colliding key returned %v, want nil", e) + } + // Original entries survive. + for _, k := range []string{"a", "b", "c"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%s should survive delete of missing colliding key", k) + } + } +} + +// TestTrie_DeleteRaceRetries stresses deleteEntry's lock-then-retry paths: +// the "parent is dead" retry and the "slot became nil after lock" +// early-return. Both fire when a concurrent delete prunes or empties the +// slot between our unlocked walk and our mu.Lock. +// +// The stress needs sustained concurrent deletes on many hash-adjacent +// keys so that pruning and slot clearing happen continuously. +func TestTrie_DeleteRaceRetries(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const workers = 16 + const keySpace = 256 + for range 200 { + var tr trie[int, int] + // Prime: every key is present. + for i := range keySpace { + triePut(&tr, i, i) + } + var wg sync.WaitGroup + wg.Add(workers * 2) + // Deleter swarms. + for w := range workers { + go func(w int) { + defer wg.Done() + for i := w; i < keySpace; i += workers { + tr.deleteEntry(i) + } + }(w) + } + // Concurrent deleter attempting same keys (likely to race on lock). + for w := range workers { + go func(w int) { + defer wg.Done() + for i := keySpace - 1 - w; i >= 0; i -= workers { + tr.deleteEntry(i) + } + }(w) + } + wg.Wait() + } +} + +// TestTrie_DeleteDuringConcurrentStore: a goroutine repeatedly deletes key +// K while another repeatedly stores it. Both succeed; the final state is +// "either present or absent" and neither crashes. +func TestTrie_DeleteDuringConcurrentStore(t *testing.T) { + var tr trie[int, int] + var stop atomic.Bool + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for !stop.Load() { + triePut(&tr, 7, 42) + } + }() + go func() { + defer wg.Done() + for !stop.Load() { + tr.deleteEntry(7) + } + }() + for range 100_000 { + trieGet(&tr, 7) + } + stop.Store(true) + wg.Wait() +} From 10c3ee784cd53afebac6c29abe6d05e4634f7661 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Tue, 21 Apr 2026 14:14:14 -0600 Subject: [PATCH 04/21] cache: back Cache with the hash-trie; retire read/dirty machinery The read/dirty/promote structure is replaced by the concurrent hash-trie from the previous commit. Public API, stale/TTL semantics, singleflight collapsing, and CompareAndSwap/Delete behavior are preserved. Shape of the rewrite: - Cache holds a single `trie[K, loading[V]]` instead of r/mu/dirty/ misses/pd. - An `ent[K, V]` type alias (Go 1.24 generic aliases) names the trie's concrete entry type so callers can read `*ent[K, V]` instead of `*trieEntry[K, loading[V]]` everywhere. All entry-manipulating logic (entGet, entTryGet, entDel, entLoad, entMaybeNewStale) is free functions over that alias, since Go generics do not permit methods on instantiation-specific aliases. - Delete physically removes the entry via the trie's deleteEntryIf primitive (with a "still tombstoned under lock" predicate to avoid clobbering a concurrent Swap resurrection). This lets keys and their referents be GC'd, preserving TestIssue40999. - Clean walks the trie, tombstones expired entries, and then calls deleteEntryIf for each tombstone so the trie's memory footprint does not grow unboundedly with delete churn. - Range iterates via the trie walk; no snapshot (matches sync.Map and the existing Range docstring). - Swap's fast path CAS on an existing entry's value slot is preserved without any global lock; the slow path goes through loadOrStoreEntry and resolves races on the trie's per-bucket mutex. Added to the trie for Cache's benefit: - deleteEntryIf(k, pred): removes only if pred returns true under the parent bucket's lock. Callers use this to avoid the resurrected-during-delete race. - Inline chain removal in deleteEntryIf, retiring the separate trieRemoveFromChain helper (it had a "not found" return path that became dead after the pred-and-locate restructure). Two new tests beyond the existing suite: - TestGet_ConcurrentStaleDuringInFlight: a second Get observes a stale-returning loading that is already in flight from the first Get and returns the stale without attempting its own miss. - TestGet_StaleRefreshCASRace: 16 concurrent Gets on an expired-with- stale key exercise the slow-path CAS-retry that replaces a finalized prev loading with a fresh loading carrying a stale snapshot. - TestClean_SkipsInFlightLoads: Clean must skip entries whose load has not finalized. - TestTrie_DeleteRaceAgainstExpand: deleteEntryIf's "slot changed to non-entry after lock" branch, triggered by concurrent insert on hash-prefix-colliding keys forcing an expand during delete. Full suite passes under -race; coverage settles at 98.6%-99.0% of statements across independent runs. The remaining uncovered lines are documented defensive panics (invariant violations that require corruption to fire) and two tightly-timed race branches that need scheduler interleavings we cannot force from userspace without test hooks. Benchmark comparison against the pre-trie baseline is deferred to the cleanup commit. --- cache/cache.go | 561 ++++++++++++++++---------------------- cache/concurrency_test.go | 97 +++++++ cache/trie.go | 76 ++++-- cache/trie_test.go | 45 +++ 4 files changed, 428 insertions(+), 351 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 1055884..8ca916d 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -1,9 +1,10 @@ // Package cache provides a concurrency safe, mostly lock-free, singleflight // request collapsing generic cache with support for stale values. // -// The guts of this package are similar to `sync.Map`, with minor differences -// in when the internal locked write-map is promoted an atomic read-only map, -// and major differences to support singleflight request collapsing. +// The internal storage is a concurrent hash-trie (see trie.go), inspired by +// Go's internal/sync.HashTrieMap. Loads are lock-free; inserts and deletes +// take a per-bucket (not global) mutex. The cache layers singleflight, +// stale values, TTLs, idle-age extension, and error caching on top. // // Functions in this API that return a KeyState return the state last, rather // than the error last: the value and error are cached as a single internal @@ -16,7 +17,6 @@ package cache import ( - "maps" "sync" "sync/atomic" "time" @@ -59,28 +59,15 @@ type ( wg sync.WaitGroup mu sync.Mutex } - ent[V any] struct { - p atomic.Pointer[loading[V]] - } - read[K comparable, V any] struct { - m map[K]*ent[V] - incomplete bool - } // Cache caches comparable keys to arbitrary values. By default, the // cache grows without bounds and all keys persist forever. These // limits can be changed with options that are passed to New. Cache[K comparable, V any] struct { - r atomic.Pointer[read[K, V]] - mu sync.Mutex - dirty map[K]*ent[V] - - misses int + t trie[K, loading[V]] cfg cfg - pd *loading[V] // allocate the promotingDelete pointer once - quitOnce sync.Once quitClean chan struct{} } @@ -106,9 +93,13 @@ type ( const ( stateLoading uint32 = iota stateFinalized - statePromotingDelete ) +// ent is the concrete type of cache entries living in the trie. Using a +// type alias (Go 1.24+) lets us refer to entries by a short name while the +// trie stays a generic hash-trie with no cache-specific logic. +type ent[K comparable, V any] = trieEntry[K, loading[V]] + func now() int64 { return time.Now().UnixNano() } func (cfg *cfg) newExpires(err error) int64 { @@ -192,9 +183,7 @@ func New[K comparable, V any](opts ...Opt) *Cache[K, V] { } c := &Cache[K, V]{ cfg: cfg, - pd: new(loading[V]), } - c.pd.state.Store(statePromotingDelete) if cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { c.quitClean = make(chan struct{}) @@ -219,116 +208,130 @@ func New[K comparable, V any](opts ...Opt) *Cache[K, V] { // cached value has an error, and there is an unexpired stale value, this // returns the stale value and no error. func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeyState) { - r := c.read() - e := r.m[k] - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - return v, err, s - } - - // We missed in the read map. We lock and check again to guard against - // something concurrent. Odds are we are falling into the logic below. - c.mu.Lock() - r = c.read() - e = r.m[k] - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - c.mu.Unlock() - return v, err, s - } - - // We could have an entry in our read map that was deleted and has not - // yet gone through the promote&clear process. We only check the dirty - // map if the entry is nil. - if e == nil && r.incomplete { - e = c.dirty[k] - c.missed(r) - r = c.read() - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - c.mu.Unlock() + // Fast path: if the entry already exists and holds a live value, use + // it without any locking. + if e := c.t.loadEntry(k); e != nil { + if v, err, s = entGet(e, c.cfg.maxIdleAge); s == Hit { return v, err, s } } - l := &loading[V]{ - stale: e.maybeNewStale(c.cfg.maxStaleAge), - } - l.wg.Add(1) + // Slow path: acquire or create the entry and install a fresh loading + // if it has no live value. loadOrStoreEntry takes the trie's per-bucket + // lock for the insert/replace window. + e, _ := c.t.loadOrStoreEntry(k, nil) - // If we have no entry, this is completely new. If we had an entry, it - // was in the read map and not yet deleted through promotion. We know - // the pointer is not promotingDelete, since that is only set while - // promoting which requires the cache lock which we have right now. - // - // In the worst case we race with a concurrent Set and we override its - // results. - if e != nil { - e.p.Store(l) - } else { - e = new(ent[V]) - e.p.Store(l) - c.storeDirty(r, k, e) + // Either the entry is newly created (p=nil), tombstoned (p=nil from a + // prior Delete), or holds a live loading from a concurrent Get/Set. We + // loop to handle the race where p is non-nil but expired/errored — we + // want to replace it with a fresh loading and carry forward a stale. + var l *loading[V] + for { + prev := e.p.Load() + if prev == nil { + l = &loading[V]{} + l.wg.Add(1) + if e.p.CompareAndSwap(nil, l) { + break + } + continue + } + // There is an existing value. Use entGet to decide whether to + // wait for it (finalized hit), return a stale (finalized but + // expired/errored with a stale), or replace (expired with no + // stale). For the replace case, we need a fresh loading whose + // stale carries the old value. + if v, err, s = entGet(e, c.cfg.maxIdleAge); s == Hit { + return v, err, s + } + if s == Stale { + // We already have a valid stale to return; a concurrent + // goroutine is (or will be) driving the load, and we can + // piggyback. But we must ensure *someone* is driving the + // load — if the prev loading is finalized but expired and + // has a stale, entGet returned Stale but nothing is yet + // refreshing. Install a fresh loading now. + // + // Concretely: if prev is finalized, we race to replace it. + // If prev is still loading (not finalized), another Get + // already triggered a miss; entGet returned prev.stale and + // that caller will finalize. + if prev.finalized() { + l = &loading[V]{ + stale: entMaybeNewStale(e, c.cfg.maxStaleAge), + } + l.wg.Add(1) + if !e.p.CompareAndSwap(prev, l) { + continue + } + break + } + // Loading in flight; return the stale from entGet. + return v, err, s + } + // Miss: prev is finalized, expired, and has no valid stale (or + // prev is tombstoned and slipped through the nil check above via + // race). Install a fresh loading with a stale snapshot from prev. + l = &loading[V]{ + stale: entMaybeNewStale(e, c.cfg.maxStaleAge), + } + l.wg.Add(1) + if e.p.CompareAndSwap(prev, l) { + break + } } - c.mu.Unlock() go func() { v, err := miss(); l.setve(v, err, c.cfg.newExpires(err)) }() - // We could have set our own stale value which can be returned - // immediately rather than waiting for the get. If there is a valid - // stale to be returned now, `get` returns it. If `get` returns a Hit, - // we ended up waiting for ourself and we return a miss. There should - // be no Miss returns from `get`. - v, err, s = e.get(c.cfg.maxIdleAge) + // Wait for our loading to finalize, then return the value. Waiting via + // l.wg (our own) synchronizes with whoever called wg.Done — the miss + // goroutine's setve or a racing Swap's defer — even if e.p has since + // been replaced by another writer. + v, err, s = entGet(e, c.cfg.maxIdleAge) switch s { case Miss, Hit: - // We always return l.v and l.err. If this expires immediately, - // get may return no value. We want to return at least the - // value generated from this miss. - // - // If e.get above waited on a different loading (because a - // concurrent Swap/Set replaced e.p while we were racing), our - // l.v may still be unsynchronized with whoever called l.wg.Done - // — either the miss goroutine's setve or Swap's defer block, - // both of which write l.v before calling Done. Waiting on - // l.wg here provides the happens-before edge we need. l.wg.Wait() return l.v, l.err, Miss } return v, err, Stale } -func (c *Cache[K, V]) tryLoadEnt(k K, dirty func()) *ent[V] { - r := c.read() - e := r.m[k] - if e == nil && r.incomplete { - c.mu.Lock() - r = c.read() - e = r.m[k] - if e == nil && r.incomplete { - e = c.dirty[k] - if dirty != nil { - dirty() - } - c.missed(r) - } - c.mu.Unlock() - } - return e -} - // TryGet returns the value for the given key if it is cached. This returns // either the currently stored value, or the current stale if the load errored, // or the load error if there is no stale. If nothing is cached, or what is // cached is expired, this returns Miss. -func (c *Cache[K, V]) TryGet(k K) (V, error, KeyState) { - e := c.tryLoadEnt(k, nil) - return e.tryGet(0, c.cfg.maxIdleAge) +func (c *Cache[K, V]) TryGet(k K) (v V, err error, _ KeyState) { + e := c.t.loadEntry(k) + if e == nil { + return v, err, Miss + } + return entTryGet(e, 0, c.cfg.maxIdleAge) } // Delete deletes the value for a key and returns the prior value, if stored // (i.e. the return from TryGet). -func (c *Cache[K, V]) Delete(k K) (V, error, KeyState) { - e := c.tryLoadEnt(k, func() { delete(c.dirty, k) }) - defer e.del() - return e.tryGet(0, 0) +// +// Delete physically removes the entry from the trie. If a concurrent Get is +// holding a reference to the deleted entry via its in-flight loading, that +// Get still completes with the miss function's result; a new Get arriving +// after Delete creates a fresh entry and may drive an independent miss. +func (c *Cache[K, V]) Delete(k K) (v V, err error, _ KeyState) { + e := c.t.loadEntry(k) + if e == nil { + return v, err, Miss + } + v, err, s := entTryGet(e, 0, 0) + // Tombstone the value slot first so any lingering in-flight readers + // see nothing. Then physically remove the entry from the trie so that + // the key (and any memory it refers to) is eligible for GC. + entDel(e) + c.t.deleteEntryIf(k, func(ee *ent[K, V]) bool { + // Only remove if nobody resurrected the slot under the lock. A + // concurrent Swap between entDel above and the lock below may + // have installed a fresh loading; leave that entry alone. + return ee.p.Load() == nil + }) + return v, err, s } // Expire sets a stored value to expire immediately, meaning the next Get will @@ -339,8 +342,11 @@ func (c *Cache[K, V]) Delete(k K) (V, error, KeyState) { // key whose miss function is still running is a no-op; the in-flight load will // complete with its normal TTL and is not canceled or shortened. func (c *Cache[K, V]) Expire(k K) { - e := c.tryLoadEnt(k, nil) - if l := e.load(); l != nil && l.finalized() { + e := c.t.loadEntry(k) + if e == nil { + return + } + if l := entLoad(e); l != nil && l.finalized() { l.expires.Store(now() - 1) } } @@ -349,69 +355,65 @@ func (c *Cache[K, V]) Expire(k K) { func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // When ranging, repeated time.Now() calls add up, so we get the // current time when we enter range and avoid it in all tryGet calls. - now := now() - c.each(func(k K, e *ent[V]) bool { - v, err, s := e.tryGet(now, 0) + tn := now() + c.t.walk(func(e *ent[K, V]) bool { + v, err, s := entTryGet(e, tn, 0) if s.IsMiss() { return true } - return fn(k, v, err) + return fn(e.key, v, err) }) } -// Similar to sync.Map, we promote on range because range is O(N) (usually) and -// this amortizes out. -func (c *Cache[K, V]) each(fn func(K, *ent[V]) bool) { - r := c.read() - if r.incomplete { - c.mu.Lock() - r = c.read() - if r.incomplete { - c.promote() - r = c.read() - } - c.mu.Unlock() - } - for k, e := range r.m { - if !fn(k, e) { - return - } - } -} - // Clean deletes all expired values from the cache. A value is expired if // MaxAge is used and the entry is older than the max age, or if you manually // expired a key. If MaxStaleAge is used and not -1, the entry must be older // than MaxAge + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. +// +// Clean also physically removes entries that were tombstoned by prior Delete +// calls, so the trie's memory footprint does not grow unboundedly with +// delete churn. func (c *Cache[K, V]) Clean() { - // If MaxStaleAge is -1, the user opted into persisting stales forever - // and we do not clean. if c.cfg.maxStaleAge < 0 { return } - now := now() - c.each(func(_ K, e *ent[V]) bool { - if l := e.load(); l != nil && l.finalized() { - expires := l.expires.Load() - if expires != 0 && now > expires+int64(c.cfg.maxStaleAge) { - // c.each promotes when necessary, so every entry - // we see lives in the read map. e.del alone is - // sufficient; no dirty bookkeeping or per-key - // relock is required. - e.del() + tn := now() + + // First pass: tombstone expired entries. Collect keys that need + // physical removal (both newly-tombstoned and pre-existing tombstones). + var toPrune []K + c.t.walk(func(e *ent[K, V]) bool { + l := e.p.Load() + if l == nil { + toPrune = append(toPrune, e.key) + return true + } + if !l.finalized() { + return true + } + expires := l.expires.Load() + if expires != 0 && tn > expires+int64(c.cfg.maxStaleAge) { + if e.p.CompareAndSwap(l, nil) { + toPrune = append(toPrune, e.key) } } return true }) + + // Second pass: remove tombstoned entries, but only if they are still + // tombstoned under the trie's bucket lock. A concurrent Swap may have + // resurrected the entry with a fresh loading in the meantime; in that + // case we leave it alone. + for _, k := range toPrune { + c.t.deleteEntryIf(k, func(e *ent[K, V]) bool { + return e.p.Load() == nil + }) + } } // Clear deletes all keys from the cache, resetting it to an empty state. func (c *Cache[K, V]) Clear() { - c.mu.Lock() - c.dirty = nil - c.storeRead(read[K, V]{}) - c.misses = 0 - c.mu.Unlock() + c.t.clear() } // StopAutoClean stops the auto clean goroutine that is began with the @@ -440,11 +442,11 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { // The following block is similar to setve, but expanded to // return our prior value (or stale, or err) if it exists. - now := now() + n := now() if !was.finalized() { was.mu.Lock() if !was.finalized() { - if was.stale != nil && !was.stale.expired(now) { + if was.stale != nil && !was.stale.expired(n) { old, oldState = was.stale.v, Stale } was.v = v @@ -456,8 +458,8 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { } was.mu.Unlock() } - if expired := was.expired(now); was.err != nil || expired { - if was.stale != nil && !was.stale.expired(now) { + if expired := was.expired(n); was.err != nil || expired { + if was.stale != nil && !was.stale.expired(n) { old, oldState = was.stale.v, Stale } if expired { @@ -467,33 +469,36 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { old, oldErr, oldState = was.v, was.err, Hit }() - r := c.read() - if e, ok := r.m[k]; ok { - for { - rm := e.p.Load() - if rm != nil && rm.promotingDelete() { // deleted & currently being ignored in a promote - break - } - was = rm - if e.p.CompareAndSwap(rm, l) { - return old, oldErr, oldState - } + // Fast path: if the entry already exists, CAS the value slot directly + // without going through the trie's bucket lock. + if e := c.t.loadEntry(k); e != nil { + rm := e.p.Load() + was = rm + if e.p.CompareAndSwap(rm, l) { + return old, oldErr, oldState } + // Lost the CAS; fall into the slow path where we can reason + // about the entry under the trie's bucket lock. } - c.mu.Lock() - r = c.read() - if e := r.m[k]; e != nil { - was = e.p.Swap(l) // was not in read, but promoted by the time we entered the lock and is now in read - } else if e = c.dirty[k]; e != nil { - was = e.p.Swap(l) - } else { - e := new(ent[V]) - e.p.Store(l) - c.storeDirty(r, k, e) + // Slow path: get or create an entry and swap in our value. The trie's + // bucket lock ensures creation is exclusive; Swap on the value slot is + // atomic. + e, _ := c.t.loadOrStoreEntry(k, l) + // If the entry was freshly created with value=l, we're done. Otherwise + // atomically swap in l and capture the previous loading. + for { + rm := e.p.Load() + if rm == l { + // We are the entry's installer; nothing was there before. + was = nil + return old, oldErr, oldState + } + was = rm + if e.p.CompareAndSwap(rm, l) { + return old, oldErr, oldState + } } - c.mu.Unlock() - return old, oldErr, oldState } func (l *loading[V]) setve(v V, err error, expires int64) { @@ -520,46 +525,24 @@ func (c *Cache[K, V]) Set(k K, v V) { // CompareAndSwap swaps the old and new values for k if the value has finished // loading and the value is equal to old. The type V must be comparable. func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { - r := c.read() - if e, ok := r.m[k]; ok { - return c.tryCAS(e, old, new, true) - } else if !r.incomplete { + e := c.t.loadEntry(k) + if e == nil { return false } - c.mu.Lock() - defer c.mu.Unlock() - r = c.read() - if e, ok := r.m[k]; ok { - return c.tryCAS(e, old, new, true) - } else if e, ok := c.dirty[k]; ok { - defer c.missed(r) - return c.tryCAS(e, old, new, true) - } - return false + return c.tryCAS(e, old, new, true) } // CompareAndDelete deletes the entry for k if the value has finished loading // and the value is equal to old. The type V must be comparable. -func (c *Cache[K, V]) CompareAndDelete(k K, old V) (deleted bool) { - r := c.read() - e, ok := r.m[k] - if !ok && r.incomplete { - c.mu.Lock() - r = c.read() - e, ok = r.m[k] - if !ok && r.incomplete { - e, ok = c.dirty[k] - c.missed(r) - } - c.mu.Unlock() - } - if ok { - return c.tryCAS(e, old, old, false) +func (c *Cache[K, V]) CompareAndDelete(k K, old V) bool { + e := c.t.loadEntry(k) + if e == nil { + return false } - return false + return c.tryCAS(e, old, old, false) } -func (c *Cache[K, V]) tryCAS(e *ent[V], old, new V, useNew bool) bool { +func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { l := e.p.Load() if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { return false @@ -579,71 +562,9 @@ func (c *Cache[K, V]) tryCAS(e *ent[V], old, new V, useNew bool) bool { } } -////////////////////// -// CACHE READ/DIRTY // -////////////////////// - -func (c *Cache[K, V]) read() read[K, V] { - p := c.r.Load() - if p == nil { - return read[K, V]{} - } - return *p -} -func (c *Cache[K, V]) storeRead(r read[K, V]) { c.r.Store(&r) } - -func (c *Cache[K, V]) storeDirty(r read[K, V], k K, e *ent[V]) { - if !r.incomplete { - if c.dirty == nil { - c.dirty = make(map[K]*ent[V]) - } - c.storeRead(read[K, V]{m: r.m, incomplete: true}) - } - c.dirty[k] = e -} - -func (c *Cache[K, V]) missed(r read[K, V]) { - c.misses++ - if len(c.dirty) == 0 || c.misses > len(r.m)>>1 { - c.promote() - } -} - -func (c *Cache[K, V]) promote() { - r := c.read() - keep := r.m - - defer func() { - c.storeRead(read[K, V]{m: keep}) - c.misses = 0 - }() - - if len(c.dirty) == 0 { - return - } - - keep = make(map[K]*ent[V], len(keep)+len(c.dirty)) - maps.Copy(keep, c.dirty) - c.dirty = nil - -outer: - for k, e := range r.m { - p := e.p.Load() - for p == nil { - if e.p.CompareAndSwap(nil, c.pd) { - continue outer - } - p = e.p.Load() // concurrently deleted while promoting - } - if p != c.pd { - keep[k] = e - } - } -} - -///////// -// ENT // -///////// +/////////////////// +// ENTRY HELPERS // +/////////////////// func (c *Cache[K, V]) finalizedLoading(v V) *loading[V] { l := &loading[V]{ @@ -653,24 +574,27 @@ func (c *Cache[K, V]) finalizedLoading(v V) *loading[V] { if expires != 0 || c.cfg.maxStaleAge != 0 { l.expires.Store(expires) if c.cfg.maxStaleAge != 0 { - l.stale = newStale(v, l.expires.Load(), c.cfg.maxStaleAge) + // newStale handles expires <= 0 by basing the stale lifespan + // on now, so an infinite-TTL + MaxStaleAge combo produces a + // usable (non-epoch-born) stale for manual Expire to surface. + l.stale = newStale(v, expires, c.cfg.maxStaleAge) } } l.state.Store(stateFinalized) return l } -func (l *loading[V]) promotingDelete() bool { return l.state.Load() == 2 } -func (l *loading[V]) finalized() bool { return l.state.Load() != 0 } -func (l *loading[V]) onlyFinalized() bool { return l.state.Load() == 1 } +func (l *loading[V]) finalized() bool { return l.state.Load() != 0 } +func (l *loading[V]) onlyFinalized() bool { return l.state.Load() == 1 } -func (e *ent[V]) del() { - if e == nil { - return - } +// entDel tombstones the entry by atomically setting its value slot to nil. +// The entry itself remains in the trie; Clean later removes tombstones +// physically. This matches the old "e.del then promote" pattern so that a +// concurrent Swap can resurrect the entry by re-populating the value slot. +func entDel[K comparable, V any](e *ent[K, V]) { for { p := e.p.Load() - if p == nil || p.promotingDelete() { + if p == nil { return } if e.p.CompareAndSwap(p, nil) { @@ -679,40 +603,36 @@ func (e *ent[V]) del() { } } -func (e *ent[V]) load() *loading[V] { - if e == nil { - return nil - } - p := e.p.Load() - if p == nil || p.promotingDelete() { - return nil - } - return p +// entLoad returns the entry's current loading, or nil if the slot is +// tombstoned. Callers guarantee e is non-nil. +func entLoad[K comparable, V any](e *ent[K, V]) *loading[V] { + return e.p.Load() } -func (l *loading[V]) expired(now int64) bool { +func (l *loading[V]) expired(n int64) bool { expires := l.expires.Load() - return expires != 0 && expires <= now // 0 means either miss not resolved, or no max age + return expires != 0 && expires <= n // 0 means either miss not resolved, or no max age } -func (s *stale[V]) expired(now int64) bool { +func (s *stale[V]) expired(n int64) bool { expires := s.expires - return expires != 0 && expires <= now + return expires != 0 && expires <= n } -// If an entry is expiring, we create a new entry with this previous entry as -// a stale. +// entMaybeNewStale derives a new stale snapshot from the entry's current +// loading, for use when installing a replacement loading. // -// - if entry is nil, no stale, return nil +// - if entry has no live loading, no stale, return nil // - if no stale age, we are not using stales, return nil -// - if entry has an error, return prior stale +// - if the loading is still pending, return nil (no value to snapshot) +// - if loading has an error, return the prior stale // - if age is < 0, return new unexpiring stale // - else, return new stale with prior expiry + stale age -func (e *ent[V]) maybeNewStale(age time.Duration) *stale[V] { - if e == nil || age == 0 { +func entMaybeNewStale[K comparable, V any](e *ent[K, V], age time.Duration) *stale[V] { + if age == 0 { return nil } - l := e.load() + l := entLoad(e) if l == nil || !l.finalized() { return nil } @@ -722,7 +642,7 @@ func (e *ent[V]) maybeNewStale(age time.Duration) *stale[V] { return newStale(l.v, l.expires.Load(), age) } -// Actually returns the stale; age must be non-zero. +// newStale returns a fresh stale; age must be non-zero. // // expires is the main entry's expiry nano. If it is <= 0 (either 0 meaning // the main entry never expires, or -1 meaning it expired immediately), we @@ -738,15 +658,13 @@ func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { return &stale[V]{v, expires + int64(age)} } -// get always returns the value or the stale value. We do not check if our +// entGet always returns the value or the stale value. We do not check if our // value is expired: we call this at the end of Get, we must always return // something even if it is to be immediately expired. -func (e *ent[V]) get(extend time.Duration) (v V, err error, state KeyState) { - l := e.load() +func entGet[K comparable, V any](e *ent[K, V], extend time.Duration) (v V, err error, state KeyState) { + l := entLoad(e) var waited bool if l == nil { - // Could be nil if deleted while in the read map, or - // promotingDelete. return v, err, state } if !l.finalized() { @@ -764,30 +682,27 @@ func (e *ent[V]) get(extend time.Duration) (v V, err error, state KeyState) { // If we waited, we could immediately be expired due to time sync, or // if the user is configured to never cache and they're just using // request collapsing: we still want to return the now expired value. - now := now() - if (!waited && l.expired(now)) || l.err != nil { - if l.stale != nil && !l.stale.expired(now) { + n := now() + if (!waited && l.expired(n)) || l.err != nil { + if l.stale != nil && !l.stale.expired(n) { return l.stale.v, nil, Stale } // The stale value is expired: if our entry is not expired, // this must be an error we waited on. - if !l.expired(now) { + if !l.expired(n) { return l.v, l.err, Hit } return v, err, state } if extend > 0 && l.err == nil { - l.expires.Store(now + int64(extend)) + l.expires.Store(n + int64(extend)) } return l.v, l.err, Hit } -func (e *ent[V]) tryGet(n64 int64, extend time.Duration) (v V, err error, state KeyState) { - if e == nil { - return v, err, state - } - l := e.load() - if l == nil { // deleting or promotedDelete +func entTryGet[K comparable, V any](e *ent[K, V], n64 int64, extend time.Duration) (v V, err error, state KeyState) { + l := entLoad(e) + if l == nil { return v, err, state } // Fast path: finalized, no expiry, no error — skip time checks. @@ -797,20 +712,20 @@ func (e *ent[V]) tryGet(n64 int64, extend time.Duration) (v V, err error, state if n64 == 0 { n64 = now() } - now := n64 + n := n64 // If we are loading but there is a valid stale, return it, otherwise // return immediately: no get. if !l.finalized() { - if l.stale != nil && !l.stale.expired(now) { + if l.stale != nil && !l.stale.expired(n) { return l.stale.v, nil, Stale } return v, err, state } // If we have an error or we are expired, we maybe return the stale. - if expired := l.expired(now); l.err != nil || expired { - if l.stale != nil && !l.stale.expired(now) { + if expired := l.expired(n); l.err != nil || expired { + if l.stale != nil && !l.stale.expired(n) { return l.stale.v, nil, Stale } if expired { @@ -818,7 +733,7 @@ func (e *ent[V]) tryGet(n64 int64, extend time.Duration) (v V, err error, state } } if extend > 0 && l.err == nil { - l.expires.Store(now + int64(extend)) + l.expires.Store(n + int64(extend)) } return l.v, l.err, Hit } diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index e167dc4..c1db477 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -244,6 +244,103 @@ func TestCollapsing_NoCaching(t *testing.T) { } } +// TestGet_StaleRefreshCASRace pounds on the Get slow-path CAS-retry that +// replaces a finalized-expired-with-stale loading with a fresh loading +// whose stale is a snapshot of the old value. Many concurrent Gets on the +// same key force CAS losses and thus exercise the retry. +func TestGet_StaleRefreshCASRace(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + for range 200 { + c := New[string, int]( + MaxAge(time.Microsecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(time.Millisecond) // entry expired, stale valid + + const workers = 16 + var wg sync.WaitGroup + wg.Add(workers) + for range workers { + go func() { + defer wg.Done() + c.Get("k", func() (int, error) { + // Slow miss so concurrent Gets all pile into the + // slow-path stale branch before any finalizes. + time.Sleep(time.Millisecond) + return 2, nil + }) + }() + } + wg.Wait() + } +} + +// TestGet_ConcurrentStaleDuringInFlight exercises Get's slow-path branch +// where a second Get observes a stale-returning loading that is already +// in flight from the first Get. The second Get must return the stale +// without attempting to install a new loading. +func TestGet_ConcurrentStaleDuringInFlight(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(10 * time.Millisecond) // entry now expired, stale still valid + + missRelease := make(chan struct{}) + missStart := make(chan struct{}) + var aDone sync.WaitGroup + aDone.Add(1) + go func() { + defer aDone.Done() + c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 2, nil + }) + }() + <-missStart + + // Second Get observes the first Get's in-flight loading and its stale. + v, _, s := c.Get("k", func() (int, error) { + t.Error("second Get's miss function should not run while first Get is in flight") + return 3, nil + }) + if s != Stale || v != 1 { + t.Fatalf("second Get: v=%d s=%v, want v=1 Stale", v, s) + } + + close(missRelease) + aDone.Wait() +} + +// TestClean_SkipsInFlightLoads verifies Clean leaves pending loads alone. +// An entry whose loading hasn't finalized yet has no expiry to evaluate; +// Clean must skip it. +func TestClean_SkipsInFlightLoads(t *testing.T) { + c := New[string, int](MaxAge(time.Nanosecond)) + missRelease := make(chan struct{}) + missStarted := make(chan struct{}) + getDone := make(chan struct{}) + go func() { + defer close(getDone) + c.Get("k", func() (int, error) { + close(missStarted) + <-missRelease + return 42, nil + }) + }() + <-missStarted + // The entry exists in the trie with an in-flight loading. Clean must + // observe !l.finalized() and skip. + c.Clean() + close(missRelease) + <-getDone +} + // TestCleanUnderConcurrency stresses Clean racing with concurrent // Set/Get/Delete to catch races in the clean path. Correctness here is that // the test finishes without -race complaints or panics; we also assert that diff --git a/cache/trie.go b/cache/trie.go index 80f4bef..168b940 100644 --- a/cache/trie.go +++ b/cache/trie.go @@ -285,6 +285,18 @@ func (t *trie[K, V]) expand(oldEntry, newEntry *trieEntry[K, V], newHash uintptr // deleteEntry removes the entry for k from the trie. Returns the removed // entry (or nil). Empty interior ancestors are pruned bottom-up. func (t *trie[K, V]) deleteEntry(k K) *trieEntry[K, V] { + return t.deleteEntryIf(k, nil) +} + +// deleteEntryIf is like deleteEntry, but only proceeds if pred(entry) +// returns true when evaluated under the parent's lock. This lets callers +// guard against racing resurrections: for example, Clean wants to +// physically remove an entry whose value slot is nil, but only if the slot +// is still nil at the moment the lock is acquired (a concurrent Swap may +// have CAS'd a fresh loading in since Clean's initial walk observed nil). +// +// If pred is nil, deleteEntryIf is equivalent to unconditional deletion. +func (t *trie[K, V]) deleteEntryIf(k K, pred func(*trieEntry[K, V]) bool) *trieEntry[K, V] { if !t.inited.Load() { return nil } @@ -330,18 +342,48 @@ func (t *trie[K, V]) deleteEntry(k K) *trieEntry[K, V] { break } - // Under i.mu, remove k from the overflow chain. If k was the head and - // had no overflow, slot becomes nil; otherwise slot gets the new head. + // Under i.mu, locate the entry matching k in the overflow chain, call + // pred, and unlink in one pass. Because we pre-locate the target before + // calling pred, the subsequent removal cannot fail to find k. head := n.entry() - removed, newHead, found := trieRemoveFromChain(head, k) - if !found { + var ( + target *trieEntry[K, V] + prev *trieEntry[K, V] // non-nil when target is not the head + newHead *trieEntry[K, V] // chain after removal, nil if slot becomes empty + ) + if head.key == k { + target = head + } else { + prev = head + for { + next := prev.overflow.Load() + if next == nil { + // k is not in the chain. + i.mu.Unlock() + return nil + } + if next.key == k { + target = next + break + } + prev = next + } + } + if pred != nil && !pred(target) { i.mu.Unlock() return nil } + if prev == nil { + // Removing head. + newHead = target.overflow.Load() + } else { + prev.overflow.Store(target.overflow.Load()) + newHead = head + } if newHead != nil { slot.Store(&newHead.trieNode) i.mu.Unlock() - return removed + return target } slot.Store(nil) @@ -364,29 +406,7 @@ func (t *trie[K, V]) deleteEntry(k K) *trieEntry[K, V] { i = parent } i.mu.Unlock() - return removed -} - -// trieRemoveFromChain walks the overflow chain starting at head, removes -// the entry whose key equals k, and returns the removed entry, the new -// chain head (which may be nil if the only entry was the head), and -// whether an entry was actually removed. -func trieRemoveFromChain[K comparable, V any](head *trieEntry[K, V], k K) (removed, newHead *trieEntry[K, V], found bool) { - if head.key == k { - return head, head.overflow.Load(), true - } - prev := head - for { - next := prev.overflow.Load() - if next == nil { - return nil, head, false - } - if next.key == k { - prev.overflow.Store(next.overflow.Load()) - return next, head, true - } - prev = next - } + return target } // walk iterates every entry in the trie, calling f for each. If f returns diff --git a/cache/trie_test.go b/cache/trie_test.go index cf191d1..b64a83e 100644 --- a/cache/trie_test.go +++ b/cache/trie_test.go @@ -526,6 +526,51 @@ func TestTrie_DeleteRaceRetries(t *testing.T) { } } +// TestTrie_DeleteRaceAgainstExpand targets deleteEntryIf's "slot changed to +// non-entry after lock" branch. An indirect-node split (expand) during the +// window between delete's unlocked walk and its parent.Lock forces the +// reload to see a non-entry node. +// +// We use a controlled hashFn that collides a few keys in the top bits but +// not all bits, so inserting them triggers expand. Concurrent insert+delete +// races through the expand window. +func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + // Keys whose hashes match in the top 4 bits but diverge later: they + // share a slot at depth 1 and a leaf at depth 2 (via overflow then + // expand). + hashes := map[string]uintptr{ + "a": 0xA000000000000000, + "b": 0xA000000000000001, + "c": 0xA000000000000002, + "d": 0xB000000000000000, + } + for range 500 { + var tr trie[string, int] + tr.hashFn = func(k string) uintptr { return hashes[k] } + triePut(&tr, "a", 1) + triePut(&tr, "d", 4) + + var wg sync.WaitGroup + wg.Add(3) + go func() { + defer wg.Done() + triePut(&tr, "b", 2) + }() + go func() { + defer wg.Done() + triePut(&tr, "c", 3) + }() + go func() { + defer wg.Done() + tr.deleteEntry("a") + }() + wg.Wait() + } +} + // TestTrie_DeleteDuringConcurrentStore: a goroutine repeatedly deletes key // K while another repeatedly stores it. Both succeed; the final state is // "either present or absent" and neither crashes. From 7c63a660228b3f29cc13a06669d713ec04044388 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Tue, 21 Apr 2026 14:17:46 -0600 Subject: [PATCH 05/21] README: rewrite for the trie-backed cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the design blurb (no more read/dirty promotion), the coverage footnote (97% → ~99% plus a note on the defensive-panic residue), and the microbenchmark section (fresh numbers from the current code on Apple M1; the old table compared sync.Map to the pre-trie cache and is no longer meaningful). Flag that per-op allocations on the insert/swap paths are intrinsic — the singleflight machinery (loading[V] with a Mutex, WaitGroup, and atomic counters) is something sync.Map doesn't carry, and carrying it costs a single heap allocation per inserted key. --- README.md | 135 +++++++++++++++++++----------------------------------- 1 file changed, 48 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 64bb839..32969c7 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,13 @@ cache Package cache provides a concurrency safe, mostly lock-free, singleflight request collapsing generic cache with support for stale values. -The guts of this package are similar to `sync.Map`, with minor differences in -when the internal locked write-map is promoted an atomic read-only map, and -major differences to support singleflight request collapsing. +The internal storage is a concurrent hash-trie, modeled on Go's +`internal/sync.HashTrieMap` (the backing of the current `sync.Map`) but +implemented entirely in userspace via `hash/maphash` for the runtime's typed +hasher. Loads are lock-free; inserts and deletes take only a per-bucket +mutex, never a global one. On top of that storage the cache layers +singleflight request collapsing, stale values, TTLs, idle-age extension, and +error caching. Functions in this API that return a `KeyState` return the state last, rather than the error last: the value and error are cached as a single internal unit @@ -34,9 +38,13 @@ value can be kept and returned from `Get` during a refresh / if a refresh fails. Keys can be manually expired with `Expire`. Internally expired values or errors can be occasionally cleaned with `Clean`. -Out of an abundance of paranoia that this code is correct, there are unit tests -to hit 97% coverage of the `cache.Cache` type. As well, all tests against -`sync.Map` are copied into this library and used against `cache.Cache`. +Out of an abundance of paranoia that this code is correct, there are unit +tests targeting ~99% statement coverage of the cache and trie (the remainder +being defensive panics at invariant-violation points and two tightly-timed +race branches that require goroutine interleavings we cannot force from +userspace). All tests against `sync.Map` are copied into this library and +used against `cache.Cache`, and a property test compares the cache's behavior +to a `sync.RWMutex`-guarded `map` oracle over random operation sequences. This package also provides a cached `Item` and a `Set`. `Item` can be used to populate an expensive value once and expire or replace it when needed, similar @@ -54,87 +62,40 @@ Documentation Benchmarking ------------ -Some of the APIs are actually faster or more memory efficient than `sync.Map` -because of generics. Some are slower and require more allocs because of the -request collapsing aspect. Some benchmarks below show more CPU, but this is due -to a few slight changes in how the write (dirty) map is internally promoted to -the read map. - -The benchmarks are from the stdlib using a key and value of type int: +Microbenchmarks on an Apple M1, Go 1.26, against the `CacheMap[int]` wrapper +(`cache.Cache[int, any]` adapted to the `sync.Map` `mapInterface` so the +stdlib benchmarks apply directly): ``` -name old time/op new time/op delta -LoadMostlyHits 12.8ns ±44% 14.0ns ±50% ~ (p=0.393 n=10+10) -LoadMostlyMisses 10.7ns ±53% 4.6ns ±47% -57.31% (p=0.000 n=10+10) -LoadOrStoreBalanced 351ns ± 3% 705ns ±27% +100.68% (p=0.000 n=8+10) -LoadOrStoreUnique 663ns ±13% 1296ns ±31% +95.53% (p=0.000 n=10+10) -LoadOrStoreCollision 9.71ns ±60% 29.26ns ±46% +201.44% (p=0.000 n=10+10) -LoadAndDeleteBalanced 11.2ns ±42% 7.1ns ±48% -36.06% (p=0.035 n=10+10) -LoadAndDeleteUnique 9.19ns ±62% 4.17ns ±47% -54.59% (p=0.001 n=10+10) -LoadAndDeleteCollision 10.9ns ±41% 14.3ns ±35% +31.18% (p=0.023 n=10+10) -Range 3.73µs ±48% 5.40µs ±45% ~ (p=0.063 n=10+10) -AdversarialAlloc 298ns ±22% 259ns ±21% ~ (p=0.063 n=10+10) -AdversarialDelete 83.7ns ±14% 105.6ns ±20% ~ (p=0.052 n=10+10) -DeleteCollision 6.87ns ±71% 4.86ns ±100% ~ (p=0.424 n=10+10) -SwapCollision 201ns ±26% 266ns ±31% ~ (p=0.143 n=10+10) -SwapMostlyHits 40.3ns ±42% 59.7ns ±51% ~ (p=0.143 n=10+10) -SwapMostlyMisses 662ns ±18% 607ns ±24% -8.19% (p=0.023 n=10+10) -CompareAndSwapCollision 33.4ns ±41% 24.9ns ±65% -25.44% (p=0.023 n=10+10) -CompareAndSwapNoExistingKey 10.3ns ±46% 2.0ns ±53% -80.15% (p=0.000 n=10+10) -CompareAndSwapValueNotEqual 9.64ns ±51% 6.02ns ±95% -37.49% (p=0.019 n=10+10) -CompareAndSwapMostlyHits 46.9ns ±39% 56.8ns ±47% ~ (p=0.143 n=10+10) -CompareAndSwapMostlyMisses 22.9ns ±39% 13.4ns ±48% -41.48% (p=0.023 n=10+10) -CompareAndDeleteCollision 28.1ns ±47% 18.3ns ±37% -34.88% (p=0.019 n=10+10) -CompareAndDeleteMostlyHits 64.0ns ±38% 63.9ns ±52% ~ (p=0.165 n=10+10) -CompareAndDeleteMostlyMisses 18.3ns ±37% 8.7ns ±53% -52.65% (p=0.023 n=10+10) - -name old alloc/op new alloc/op delta -LoadMostlyHits 6.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadMostlyMisses 6.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadOrStoreBalanced 74.7B ±16% 195.3B ± 1% +161.45% (p=0.000 n=10+10) -LoadOrStoreUnique 159B ± 7% 330B ± 7% +107.15% (p=0.000 n=9+10) -LoadOrStoreCollision 0.00B 32.00B ± 0% +Inf% (p=0.000 n=10+10) -LoadAndDeleteBalanced 4.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadAndDeleteUnique 8.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadAndDeleteCollision 1.00B ± 0% 1.00B ± 0% ~ (all equal) -Range 16.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) -AdversarialAlloc 53.4B ± 3% 64.8B ± 3% +21.35% (p=0.000 n=10+10) -AdversarialDelete 22.6B ±12% 39.0B ± 0% +72.57% (p=0.000 n=10+10) -DeleteCollision 0.00B 0.00B ~ (all equal) -SwapCollision 16.0B ± 0% 80.0B ± 0% +400.00% (p=0.000 n=10+10) -SwapMostlyHits 28.0B ± 0% 86.0B ± 0% +207.14% (p=0.000 n=10+10) -SwapMostlyMisses 124B ± 0% 136B ± 1% +9.71% (p=0.000 n=10+10) -CompareAndSwapCollision 7.60B ± 8% 21.70B ±17% +185.53% (p=0.000 n=10+10) -CompareAndSwapNoExistingKey 8.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -CompareAndSwapValueNotEqual 0.00B 0.00B ~ (all equal) -CompareAndSwapMostlyHits 33.0B ± 0% 91.0B ± 0% +175.76% (p=0.000 n=10+10) -CompareAndSwapMostlyMisses 23.0B ± 0% 16.0B ± 0% -30.43% (p=0.000 n=10+10) -CompareAndDeleteCollision 0.00B 2.00B ± 0% +Inf% (p=0.000 n=10+8) -CompareAndDeleteMostlyHits 39.0B ± 0% 90.6B ± 1% +132.31% (p=0.000 n=10+10) -CompareAndDeleteMostlyMisses 16.0B ± 0% 8.0B ± 0% -50.00% (p=0.002 n=8+10) - -name old allocs/op new allocs/op delta -LoadMostlyHits 0.00 0.00 ~ (all equal) -LoadMostlyMisses 0.00 0.00 ~ (all equal) -LoadOrStoreBalanced 2.00 ± 0% 3.00 ± 0% +50.00% (p=0.000 n=10+10) -LoadOrStoreUnique 4.00 ± 0% 5.00 ± 0% +25.00% (p=0.000 n=10+10) -LoadOrStoreCollision 0.00 1.00 ± 0% +Inf% (p=0.000 n=10+10) -LoadAndDeleteBalanced 0.00 0.00 ~ (all equal) -LoadAndDeleteUnique 0.60 ±100% 0.00 -100.00% (p=0.011 n=10+10) -LoadAndDeleteCollision 0.00 0.00 ~ (all equal) -Range 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -AdversarialAlloc 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -AdversarialDelete 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -DeleteCollision 0.00 0.00 ~ (all equal) -SwapCollision 1.00 ± 0% 1.00 ± 0% ~ (all equal) -SwapMostlyHits 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) -SwapMostlyMisses 6.00 ± 0% 4.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndSwapCollision 0.00 0.00 ~ (all equal) -CompareAndSwapNoExistingKey 0.50 ±100% 0.00 -100.00% (p=0.033 n=10+10) -CompareAndSwapValueNotEqual 0.00 0.00 ~ (all equal) -CompareAndSwapMostlyHits 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndSwapMostlyMisses 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) -CompareAndDeleteCollision 0.00 0.00 ~ (all equal) -CompareAndDeleteMostlyHits 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndDeleteMostlyMisses 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) +name time/op alloc/op allocs/op +LoadMostlyHits 10.3 ns/op 0 B/op 0 allocs/op +LoadMostlyMisses 5.2 ns/op 0 B/op 0 allocs/op +LoadOrStoreBalanced 448 ns/op 151 B/op 3 allocs/op +LoadOrStoreUnique 567 ns/op 262 B/op 5 allocs/op +LoadOrStoreCollision 45.6 ns/op 32 B/op 1 allocs/op +LoadAndDeleteBalanced 4.88 ns/op 0 B/op 0 allocs/op +LoadAndDeleteUnique 2.00 ns/op 0 B/op 0 allocs/op +LoadAndDeleteCollision 20.2 ns/op 2 B/op 0 allocs/op +Range 8.64 µs/op 0 B/op 0 allocs/op +AdversarialAlloc 11.7 ns/op 0 B/op 0 allocs/op +AdversarialDelete 7.17 ns/op 0 B/op 0 allocs/op +DeleteCollision 3.69 ns/op 0 B/op 0 allocs/op +SwapCollision 243 ns/op 80 B/op 1 allocs/op +SwapMostlyHits 119 ns/op 86 B/op 1 allocs/op +SwapMostlyMisses 160 ns/op 120 B/op 2 allocs/op +CompareAndSwapCollision 13.1 ns/op 3 B/op 0 allocs/op +CompareAndSwapNoExistingKey 2.57 ns/op 0 B/op 0 allocs/op +CompareAndSwapValueNotEqual 10.6 ns/op 0 B/op 0 allocs/op +CompareAndSwapMostlyHits 128 ns/op 91 B/op 2 allocs/op +CompareAndSwapMostlyMisses 30.1 ns/op 16 B/op 1 allocs/op +CompareAndDeleteCollision 13.0 ns/op 0 B/op 0 allocs/op +CompareAndDeleteMostlyHits 112 ns/op 91 B/op 2 allocs/op +CompareAndDeleteMostlyMisses 13.3 ns/op 8 B/op 0 allocs/op ``` + +The per-op allocations on the `LoadOrStore*`, `Swap*`, and `CompareAndSwap*Hits` +rows are intrinsic: each of these operations may install a fresh `loading[V]` +in the trie's value slot, and that struct carries the singleflight machinery +(`sync.Mutex`, `sync.WaitGroup`, atomic counters) that a plain `sync.Map` does +not need. The lock-free-read paths (`Load*`, `Delete*`, `Range`, the +`Adversarial*` patterns, and the `CompareAnd*Misses` paths) are alloc-free. From 134488b644131ba4f5237f504e7490c9cd63a65f Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Tue, 21 Apr 2026 14:57:04 -0600 Subject: [PATCH 06/21] cache: port the stdlib sync.Map tests we were missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add three tests and one benchmark that exist in sync/map_test.go and sync/map_bench_test.go but were not in our copy: - TestConcurrentClear: 10 writers, 10 readers, and 10 Clear goroutines run concurrently. Correctness is no panic and no phantom keys (keys we never Stored appearing in Range after the dust settles). - TestMapClearOneAllocation: asserts Clear allocates ≤1 time. Cache's Clear replaces the trie root with one fresh indirect node, matching the sync.Map guarantee. - TestMapRangeNoAllocations: asserts Range does not allocate. The benchmarks already showed 0 B/op but an explicit AllocsPerRun test guards against a regression in the Range closure wiring. - BenchmarkClear: Clear throughput. Skipped from stdlib (intentionally): - TestMapMatchesDeepCopy: a second mapInterface oracle (DeepCopyMap). Our TestCacheMatchesRWMutex already covers the semantics; the stdlib keeps a second oracle mainly to catch sync.Map-specific promotion bugs we no longer have. - TestMapMatchesHashTrieMap: stdlib's internal hash trie as the oracle. We are the hash trie. - TestHashTrieMap{BadHash,TruncHash}: pathological-hash tests. Our TestTrie_FullHashCollision{,DeleteHead} already exercises full collisions via the hashFn hook. - BenchmarkHashTrieMapLoad[Small|Large], LoadOrStore[Large]: trie- level microbenchmarks. Redundant with the cache-level benchmarks since Cache is thin over trie. --- cache/map_bench_test.go | 12 +++++++ cache/map_test.go | 73 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/cache/map_bench_test.go b/cache/map_bench_test.go index aeec419..4c8625f 100644 --- a/cache/map_bench_test.go +++ b/cache/map_bench_test.go @@ -513,3 +513,15 @@ func BenchmarkCompareAndDeleteMostlyMisses(b *testing.B) { }, }) } + +func BenchmarkClear(b *testing.B) { + benchMap(b, bench{ + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface[int]) { + for ; pb.Next(); i++ { + k, v := i%256, i%256 + m.Clear() + m.Store(k, v) + } + }, + }) +} diff --git a/cache/map_test.go b/cache/map_test.go index 7fb15a8..9ee6683 100644 --- a/cache/map_test.go +++ b/cache/map_test.go @@ -276,3 +276,76 @@ func TestCompareAndSwap_NonExistingKey(t *testing.T) { t.Fatalf("CompareAndSwap on an non-existing key succeeded") } } + +// TestConcurrentClear — adapted from sync/map_test.go. Spawns 10 writers, +// 10 readers, and 10 Clear goroutines; correctness is that nothing panics +// or trips the race detector. +func TestConcurrentClear(t *testing.T) { + var m CacheMap[int] + + var wg sync.WaitGroup + wg.Add(30) + + for i := range 10 { + go func(k, v int) { + defer wg.Done() + m.Store(k, v) + }(i, i*10) + } + for i := range 10 { + go func(k int) { + defer wg.Done() + if _, ok := m.Load(k); ok { + _ = ok + } + }(i) + } + for range 10 { + go func() { + defer wg.Done() + m.Clear() + }() + } + + wg.Wait() + + // After all Clears and Stores have run, the map may be empty or contain a + // subset of the Stores depending on interleaving. Verify no phantom keys + // (keys we never Stored) materialized. + m.Range(func(k int, _ any) bool { + if k < 0 || k >= 10 { + t.Errorf("Range after concurrent Clear/Store saw key %d, never Stored", k) + } + return true + }) +} + +// TestMapClearOneAllocation — adapted from sync/map_test.go. Cache.Clear +// replaces the trie root with one fresh indirect node, which should be the +// only allocation. +func TestMapClearOneAllocation(t *testing.T) { + var m CacheMap[int] + // Prime so the trie has been initialized; Clear of an uninitialized + // trie still calls init which allocates more. + m.Store(0, 0) + allocs := testing.AllocsPerRun(10, func() { + m.Clear() + }) + if allocs > 1 { + t.Errorf("AllocsPerRun of Clear = %v; want 1", allocs) + } +} + +// TestMapRangeNoAllocations — adapted from sync/map_test.go. Range must not +// allocate. +func TestMapRangeNoAllocations(t *testing.T) { + var m CacheMap[int] + allocs := testing.AllocsPerRun(10, func() { + m.Range(func(k int, _ any) bool { + return true + }) + }) + if allocs > 0 { + t.Errorf("AllocsPerRun of Range = %v; want 0", allocs) + } +} From db5c03da18debbd2a1f4c75e08440612f80b038c Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Wed, 10 Jun 2026 10:07:00 -0600 Subject: [PATCH 07/21] cache: fix delete-vs-expand and tombstone-resurrection races An audit against internal/sync.HashTrieMap (which trie.go is modeled on) and the prior read/dirty implementation found three interleaving bugs, each reproduced with hammer tests before fixing: 1) trie.deleteEntryIf gave up (returned nil) when the slot it walked to was expanded into an indirect node between its unlocked walk and its lock, silently skipping a delete of a present key (3,109/200k rounds). Upstream's find() retries from the root in this case; we now do too. 2) Tombstoned entries (nil value slot) could be resurrected by Swap/Get via a bare CAS(nil, loading) that nothing serialized against Delete's and Clean's pred-then-unlink, so a concurrent Set/Swap could land in an entry that was then physically unlinked: the swap reported success (Miss = "stored") and the value vanished (66/200k rounds). The old read/dirty code was immune via the promotingDelete sentinel, which made tombstone removal a single CAS; the port lost that atomicity. A nil value slot is now terminal: no code path ever stores over it. Writers that find a tombstone unlink the dead entry and re-create the key as a fresh entry, which makes deleteEntryIf's nil check stable under the bucket lock. Delete also now derives its return value from the loading it actually displaced, so the value reported deleted is the value removed. 3) Get created its entry with loadOrStoreEntry(k, nil) and CAS'd the loading in afterwards; in that window a fresh, never-expired entry was indistinguishable from a tombstone and Clean would evict it (1/500k rounds), also breaking request collapsing for that key. Get now creates entries with the loading already in place; a linked entry never has a nil slot except as a tombstone. Also: CompareAndDelete now physically unlinks like Delete (tombstones no longer leak until Clean); NewSet/NewItem now actually honor AutoCleanInterval (Set.StopAutoClean previously stopped a goroutine that was never started) and Item gains Clean/StopAutoClean; onlyFinalized is collapsed into finalized (the third state it guarded against retired with the read/dirty machinery); stale comments describing promote/read/ dirty are rewritten; TestTrie_DeleteRaceAgainstExpand now asserts the delete succeeds (it previously only asserted no-crash, exercising bug 1 without catching it); regression tests added for 2 and 3. --- cache/cache.go | 298 +++++++++++++++++++++----------------- cache/concurrency_test.go | 187 ++++++++++++++++-------- cache/trie.go | 27 +++- cache/trie_test.go | 31 +++- cache/wrappers_test.go | 18 ++- 5 files changed, 348 insertions(+), 213 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 8ca916d..5fdcb89 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -177,18 +177,25 @@ func AutoCleanInterval(interval time.Duration) Opt { // semantics. If you do not need to configure a cache at all, the zero value // cache is valid and usable. func New[K comparable, V any](opts ...Opt) *Cache[K, V] { - var cfg cfg + c := new(Cache[K, V]) + initCache(c, opts...) + return c +} + +// initCache applies options and starts the autoclean goroutine if +// configured. It initializes in place (rather than returning a new cache) +// so that NewItem and NewSet can initialize their embedded Cache: the +// autoclean goroutine closes over c, so the configured cache cannot be +// copied after this returns. +func initCache[K comparable, V any](c *Cache[K, V], opts ...Opt) { for _, opt := range opts { - opt.apply(&cfg) - } - c := &Cache[K, V]{ - cfg: cfg, + opt.apply(&c.cfg) } - if cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { + if c.cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { c.quitClean = make(chan struct{}) go func() { - ticker := time.NewTicker(cfg.autoCleanInterval) + ticker := time.NewTicker(c.cfg.autoCleanInterval) defer ticker.Stop() for { select { @@ -200,7 +207,6 @@ func New[K comparable, V any](opts ...Opt) *Cache[K, V] { } }() } - return c } // Get returns the cache value for k, running the miss function in a goroutine @@ -210,74 +216,65 @@ func New[K comparable, V any](opts ...Opt) *Cache[K, V] { func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeyState) { // Fast path: if the entry already exists and holds a live value, use // it without any locking. - if e := c.t.loadEntry(k); e != nil { + e := c.t.loadEntry(k) + if e != nil { if v, err, s = entGet(e, c.cfg.maxIdleAge); s == Hit { return v, err, s } } - // Slow path: acquire or create the entry and install a fresh loading - // if it has no live value. loadOrStoreEntry takes the trie's per-bucket - // lock for the insert/replace window. - e, _ := c.t.loadOrStoreEntry(k, nil) - - // Either the entry is newly created (p=nil), tombstoned (p=nil from a - // prior Delete), or holds a live loading from a concurrent Get/Set. We - // loop to handle the race where p is non-nil but expired/errored — we - // want to replace it with a fresh loading and carry forward a stale. + // Slow path: acquire or create the entry, installing a fresh loading + // to drive a miss if there is no live value. A loading is always + // created before the entry that holds it is published, so a linked + // entry never holds a nil value slot; nil means the entry was + // tombstoned by Delete or Clean and is permanently dead (see entDel). var l *loading[V] +outer: for { - prev := e.p.Load() - if prev == nil { + if e == nil { l = &loading[V]{} l.wg.Add(1) - if e.p.CompareAndSwap(nil, l) { - break + var loaded bool + if e, loaded = c.t.loadOrStoreEntry(k, l); !loaded { + break // we created the entry, holding our loading } - continue } - // There is an existing value. Use entGet to decide whether to - // wait for it (finalized hit), return a stale (finalized but - // expired/errored with a stale), or replace (expired with no - // stale). For the replace case, we need a fresh loading whose - // stale carries the old value. - if v, err, s = entGet(e, c.cfg.maxIdleAge); s == Hit { - return v, err, s - } - if s == Stale { - // We already have a valid stale to return; a concurrent - // goroutine is (or will be) driving the load, and we can - // piggyback. But we must ensure *someone* is driving the - // load — if the prev loading is finalized but expired and - // has a stale, entGet returned Stale but nothing is yet - // refreshing. Install a fresh loading now. - // - // Concretely: if prev is finalized, we race to replace it. - // If prev is still loading (not finalized), another Get - // already triggered a miss; entGet returned prev.stale and - // that caller will finalize. - if prev.finalized() { - l = &loading[V]{ - stale: entMaybeNewStale(e, c.cfg.maxStaleAge), - } - l.wg.Add(1) - if !e.p.CompareAndSwap(prev, l) { - continue - } - break + for { + prev := e.p.Load() + if prev == nil { + // Tombstoned. Unlink the dead entry (idempotent: the + // predicate re-checks under the bucket lock and leaves + // a re-created live entry alone) and start over with a + // fresh entry. + c.t.deleteEntryIf(k, entDead[K, V]) + e = nil + continue outer + } + // There is an existing value. Use entGet to decide whether + // to wait for it (finalized hit), return a stale (load in + // flight with a valid stale), or replace it (finalized but + // expired or errored). + if v, err, s = entGet(e, c.cfg.maxIdleAge); s == Hit { + return v, err, s + } + if s == Stale && !prev.finalized() { + // A load is already in flight and its caller will + // finalize it; piggyback and return the stale now. + return v, err, s + } + // prev is finalized and expired or errored, possibly with a + // valid stale that nothing is refreshing (if it was in + // flight with no stale, entGet waited for it to finalize). + // Install a fresh loading carrying a stale snapshot of prev + // and drive a new load. The CAS fails if another goroutine + // got here first; re-evaluate what it installed. + l = &loading[V]{ + stale: entMaybeNewStale(e, c.cfg.maxStaleAge), + } + l.wg.Add(1) + if e.p.CompareAndSwap(prev, l) { + break outer } - // Loading in flight; return the stale from entGet. - return v, err, s - } - // Miss: prev is finalized, expired, and has no valid stale (or - // prev is tombstoned and slipped through the nil check above via - // race). Install a fresh loading with a stale snapshot from prev. - l = &loading[V]{ - stale: entMaybeNewStale(e, c.cfg.maxStaleAge), - } - l.wg.Add(1) - if e.p.CompareAndSwap(prev, l) { - break } } @@ -320,18 +317,19 @@ func (c *Cache[K, V]) Delete(k K) (v V, err error, _ KeyState) { if e == nil { return v, err, Miss } - v, err, s := entTryGet(e, 0, 0) - // Tombstone the value slot first so any lingering in-flight readers - // see nothing. Then physically remove the entry from the trie so that - // the key (and any memory it refers to) is eligible for GC. - entDel(e) - c.t.deleteEntryIf(k, func(ee *ent[K, V]) bool { - // Only remove if nobody resurrected the slot under the lock. A - // concurrent Swap between entDel above and the lock below may - // have installed a fresh loading; leave that entry alone. - return ee.p.Load() == nil - }) - return v, err, s + // Tombstone the value slot first; the captured loading is exactly what + // this Delete removed, so the returned value is the value removed. + // Then physically unlink the entry from the trie so the key (and any + // memory it refers to) is eligible for GC. The nil slot is terminal + // (see entDel), so entDead observed under the bucket lock is stable; a + // concurrent Swap or Get that finds the tombstone re-creates the key + // as a fresh entry, which the predicate leaves alone. + was := entDel(e) + c.t.deleteEntryIf(k, entDead[K, V]) + if was == nil { + return v, err, Miss + } + return loadingTryGet(was, 0, 0) } // Expire sets a stored value to expire immediately, meaning the next Get will @@ -400,14 +398,13 @@ func (c *Cache[K, V]) Clean() { return true }) - // Second pass: remove tombstoned entries, but only if they are still - // tombstoned under the trie's bucket lock. A concurrent Swap may have - // resurrected the entry with a fresh loading in the meantime; in that - // case we leave it alone. + // Second pass: physically unlink the tombstoned entries. The nil value + // slot is terminal (see entDel), so the predicate observing nil under + // the bucket lock cannot be invalidated by a concurrent writer; if the + // key was deleted and re-created in the meantime, the lookup finds the + // fresh live entry and the predicate leaves it alone. for _, k := range toPrune { - c.t.deleteEntryIf(k, func(e *ent[K, V]) bool { - return e.p.Load() == nil - }) + c.t.deleteEntryIf(k, entDead[K, V]) } } @@ -469,34 +466,44 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { old, oldErr, oldState = was.v, was.err, Hit }() - // Fast path: if the entry already exists, CAS the value slot directly - // without going through the trie's bucket lock. + // Fast path: if the entry already exists with a live value, CAS the + // value slot directly without going through the trie's bucket lock. A + // nil slot means the entry was tombstoned by Delete or Clean; it is + // never written again (see entDel), so we fall to the slow path to + // re-create the key rather than writing into the dead entry. if e := c.t.loadEntry(k); e != nil { - rm := e.p.Load() - was = rm - if e.p.CompareAndSwap(rm, l) { - return old, oldErr, oldState + if rm := e.p.Load(); rm != nil { + was = rm + if e.p.CompareAndSwap(rm, l) { + return old, oldErr, oldState + } + // Lost the CAS; fall into the slow path. } - // Lost the CAS; fall into the slow path where we can reason - // about the entry under the trie's bucket lock. } - // Slow path: get or create an entry and swap in our value. The trie's - // bucket lock ensures creation is exclusive; Swap on the value slot is - // atomic. - e, _ := c.t.loadOrStoreEntry(k, l) - // If the entry was freshly created with value=l, we're done. Otherwise - // atomically swap in l and capture the previous loading. + // Slow path: get or create an entry and swap our value in. The trie's + // bucket lock makes creation exclusive; tombstoned entries are + // unlinked and the key re-created as a fresh entry, never resurrected + // in place. for { - rm := e.p.Load() - if rm == l { + e, loaded := c.t.loadOrStoreEntry(k, l) + if !loaded { // We are the entry's installer; nothing was there before. was = nil return old, oldErr, oldState } - was = rm - if e.p.CompareAndSwap(rm, l) { - return old, oldErr, oldState + for { + rm := e.p.Load() + if rm == nil { + // Tombstoned by Delete or Clean. Unlink the dead entry + // and retry with a fresh one. + c.t.deleteEntryIf(k, entDead[K, V]) + break + } + was = rm + if e.p.CompareAndSwap(rm, l) { + return old, oldErr, oldState + } } } } @@ -539,12 +546,18 @@ func (c *Cache[K, V]) CompareAndDelete(k K, old V) bool { if e == nil { return false } - return c.tryCAS(e, old, old, false) + if !c.tryCAS(e, old, old, false) { + return false + } + // The successful CAS tombstoned the value slot; physically unlink the + // entry like Delete does, so delete churn does not leak entries. + c.t.deleteEntryIf(k, entDead[K, V]) + return true } func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { l := e.p.Load() - if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { + if l == nil || !l.finalized() || any(l.v) != any(old) { return false } var l2 *loading[V] @@ -556,7 +569,7 @@ func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { return true } l = e.p.Load() - if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { + if l == nil || !l.finalized() || any(l.v) != any(old) { return false } } @@ -584,25 +597,35 @@ func (c *Cache[K, V]) finalizedLoading(v V) *loading[V] { return l } -func (l *loading[V]) finalized() bool { return l.state.Load() != 0 } -func (l *loading[V]) onlyFinalized() bool { return l.state.Load() == 1 } +func (l *loading[V]) finalized() bool { return l.state.Load() != 0 } -// entDel tombstones the entry by atomically setting its value slot to nil. -// The entry itself remains in the trie; Clean later removes tombstones -// physically. This matches the old "e.del then promote" pattern so that a -// concurrent Swap can resurrect the entry by re-populating the value slot. -func entDel[K comparable, V any](e *ent[K, V]) { +// entDel tombstones the entry by atomically setting its value slot to nil, +// returning the loading it displaced (nil if the entry was already +// tombstoned). +// +// A nil value slot is terminal: no code path ever stores a non-nil loading +// over it. Writers that encounter a tombstone unlink the dead entry +// (deleteEntryIf with entDead) and re-create the key as a fresh entry. This +// is what makes physical unlinking sound: entDead evaluated under the +// trie's bucket lock cannot be invalidated by a concurrent lock-free CAS, +// so an unlinked entry is always dead and a live entry is never unlinked. +func entDel[K comparable, V any](e *ent[K, V]) *loading[V] { for { p := e.p.Load() if p == nil { - return + return nil } if e.p.CompareAndSwap(p, nil) { - return + return p } } } +// entDead reports whether the entry is tombstoned. Used as a deleteEntryIf +// predicate: nil value slots are terminal (see entDel), so a nil observed +// under the bucket lock is stable. +func entDead[K comparable, V any](e *ent[K, V]) bool { return e.p.Load() == nil } + // entLoad returns the entry's current loading, or nil if the slot is // tombstoned. Callers guarantee e is non-nil. func entLoad[K comparable, V any](e *ent[K, V]) *loading[V] { @@ -705,8 +728,15 @@ func entTryGet[K comparable, V any](e *ent[K, V], n64 int64, extend time.Duratio if l == nil { return v, err, state } + return loadingTryGet(l, n64, extend) +} + +// loadingTryGet is entTryGet for a loading already plucked from an entry; +// Delete uses it directly on the loading it captured while tombstoning, so +// the value it returns is exactly the value it removed. +func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, err error, state KeyState) { // Fast path: finalized, no expiry, no error — skip time checks. - if l.onlyFinalized() && l.expires.Load() == 0 && l.err == nil { + if l.finalized() && l.expires.Load() == 0 && l.err == nil { return l.v, nil, Hit } if n64 == 0 { @@ -753,15 +783,9 @@ type Item[V any] struct { // semantics. If you do not need to configure an item at all, the zero value // item is valid and usable. func NewItem[V any](opts ...Opt) *Item[V] { - var c cfg - for _, opt := range opts { - opt.apply(&c) - } - return &Item[V]{ - c: Cache[struct{}, V]{ - cfg: c, - }, - } + i := new(Item[V]) + initCache(&i.c, opts...) + return i } // Get returns the currently cached value, running the miss function in a @@ -829,6 +853,20 @@ func (i *Item[V]) Clear() { i.c.Clear() } +// Clean deletes the item if it is expired. The item is expired if MaxAge is +// used and the item is older than the max age, or if you manually expired +// it. If MaxStaleAge is used and not -1, the item must be older than MaxAge +// + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. +func (i *Item[V]) Clean() { + i.c.Clean() +} + +// StopAutoClean stops the auto clean goroutine that is began with the +// AutoClean option. +func (i *Item[V]) StopAutoClean() { + i.c.StopAutoClean() +} + ///////// // SET // ///////// @@ -844,15 +882,9 @@ type Set[K comparable] struct { // semantics. If you do not need to configure an set at all, the zero value set // is valid and usable. func NewSet[K comparable](opts ...Opt) *Set[K] { - var c cfg - for _, opt := range opts { - opt.apply(&c) - } - return &Set[K]{ - c: Cache[K, struct{}]{ - cfg: c, - }, - } + s := new(Set[K]) + initCache(&s.c, opts...) + return s } // Get ensures the key is cached, running the miss function in a goroutine if diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index c1db477..5417c45 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -61,8 +61,6 @@ func TestSwap_CancelsInFlightGet(t *testing.T) { // TestSwap_OverExpiredFinalizedWithStale exercises the defer branch that // reads the prior entry's stale when the prior entry has expired but a valid // stale is still in its window. -// -// Covers cache.go Swap defer lines for expired-with-stale (lines 451-457). func TestSwap_OverExpiredFinalizedWithStale(t *testing.T) { c := New[string, int]( MaxAge(5*time.Millisecond), @@ -179,7 +177,7 @@ func TestExpire_NoOpDuringInFlightLoad(t *testing.T) { // queries for the same key. // // The guarantee is weaker than it looks: with del0, the loading finalizes -// expired, so the *next* goroutine that acquires c.mu after the leader's +// expired, so the next goroutine that re-checks the entry after the leader's // setve sees an expired entry and triggers a fresh miss. Concurrent arrivals // still collapse in batches, but "batch" here means "whatever queued on a // given loading's WaitGroup before it was Done'd," which is scheduler- @@ -418,19 +416,13 @@ func TestCleanUnderConcurrency(t *testing.T) { wg.Wait() } -// TestSwap_OverPromotingDelete forces two rare branches simultaneously: -// -// 1. Swap's unlocked fast path observes an entry whose pointer is the -// promotingDelete sentinel (cache.go:474-475) and falls through to the -// locked path. -// 2. promote's inner CAS(nil, pd) fails because a concurrent Swap wrote a -// fresh loading into e.p between promote's Load and CAS (cache.go:636). -// -// Both windows are ~nanoseconds wide; we stress with many deleters + many -// swappers + an aggressive Range driver for long enough that both branches -// land. On a fast machine, either can land within a few hundred ms, but we -// leave a generous budget for slow CI runners. -func TestSwap_OverPromotingDelete(t *testing.T) { +// TestSwap_DeleteRangeChurn stresses Swap's unlocked fast path racing +// Delete's tombstone-then-unlink, with Range walking the trie throughout: +// Swap must observe tombstones and re-create keys (never resurrect a dying +// entry), and Range must tolerate entries being unlinked mid-walk. The +// interesting windows are ~nanoseconds wide; we stress from many angles for +// long enough that they land, leaving a generous budget for slow CI runners. +func TestSwap_DeleteRangeChurn(t *testing.T) { if testing.Short() { t.Skip("skipping stress test in -short") } @@ -445,8 +437,8 @@ func TestSwap_OverPromotingDelete(t *testing.T) { var wg sync.WaitGroup var stop atomic.Bool - // Deleters churn the primed keys, repeatedly creating the nil-e.p window - // that promote's CAS(nil, pd) targets. + // Deleters churn the primed keys, repeatedly creating the tombstone + + // unlink windows that Swap's fast path must handle. for range 8 { wg.Add(1) go func() { @@ -459,7 +451,7 @@ func TestSwap_OverPromotingDelete(t *testing.T) { }() } // Swappers on the same primed keys exercise the unlocked fast-path CAS - // that races with promote. + // that races with the deleters. for w := range 8 { wg.Add(1) go func(w int) { @@ -471,10 +463,9 @@ func TestSwap_OverPromotingDelete(t *testing.T) { } }(w) } - // Churners add+delete NEW keys continuously so that dirty stays populated, - // which keeps r.incomplete=true and forces Range to call promote on every - // iteration. Without this, the initial promote fires once and subsequent - // Ranges skip the work. + // Churners add+delete NEW keys continuously so the trie's shape keeps + // changing (inserts expand slots, deletes prune empty parents) while + // Range walks it. for w := range 4 { wg.Add(1) go func(w int) { @@ -488,7 +479,7 @@ func TestSwap_OverPromotingDelete(t *testing.T) { } }(w) } - // Range callers hammer promote. + // Range callers walk the trie throughout the churn. for range 4 { wg.Add(1) go func() { @@ -621,16 +612,11 @@ func TestSetve_RacesWithSwap(t *testing.T) { } } -// TestPromote_ConcurrentDeleteOfNilEntry forces promote to observe an entry -// whose pointer is nil but becomes non-nil between the Load and the -// CompareAndSwap(nil, pd) inside promote's retry loop. Covers the reload -// statement inside promote. -// -// This exercises cache.go:promote retry loop. Deterministic coverage of the -// exact reload line requires a goroutine interleaving that we cannot force -// from the outside; we stress via concurrent Delete/Set/Range to try to hit -// it. -func TestPromote_ConcurrentDeleteOfNilEntry(t *testing.T) { +// TestRange_DeleteSetChurn races Range against a goroutine that +// continuously Deletes and re-Sets the same keys: every key cycles through +// tombstone, physical unlink, and fresh re-creation while the walk is in +// flight. Correctness is no panic and no race-detector report. +func TestRange_DeleteSetChurn(t *testing.T) { if testing.Short() { t.Skip("skipping stress test in -short") } @@ -677,16 +663,12 @@ func TestClean_NoopWhenStalesArePermanent(t *testing.T) { } } -// TestGet_UnlockedMissThenLockedHit targets Get's locked-path Hit (cache.go -// lines 233-236): unlocked e.get returned Miss, but by the time we acquire -// c.mu and re-check, a concurrent Swap has published a fresh value for the -// same key in the read map. -// -// Requires the entry to live in the read map (not dirty). We force that by -// priming + Range (promote), then Expire to make the unlocked fast-path Miss, -// then race a Swap against a Get. Hitting the exact timing window requires -// multiple iterations. -func TestGet_UnlockedMissThenLockedHit(t *testing.T) { +// TestGet_RacesSwapOnExpired targets Get's slow-path Hit: the fast path +// sees an expired entry (Miss), but by the time the slow path re-checks, a +// concurrent Swap has published a fresh value for the same key. If Get +// reports a Hit it must be the Swap's value, never the expired one. Hitting +// the exact timing window requires multiple iterations. +func TestGet_RacesSwapOnExpired(t *testing.T) { if testing.Short() { t.Skip("skipping race-iteration test in -short") } @@ -713,10 +695,11 @@ func TestGet_UnlockedMissThenLockedHit(t *testing.T) { } } -// TestCompareAndSwap_LockPromotedEntry targets the CompareAndSwap locked -// branch where the entry was only observable via dirty on fast path but has -// since been promoted into the read map (cache.go lines 532-534). Race window. -func TestCompareAndSwap_LockPromotedEntry(t *testing.T) { +// TestCompareAndSwap_RacesSet races CompareAndSwap against the first Set of +// the same key (plus a Range walk): the CAS may observe no entry, an +// in-flight entry, or the finalized value, and must succeed only in the +// last case. Race window; stress only. +func TestCompareAndSwap_RacesSet(t *testing.T) { if testing.Short() { t.Skip("skipping race-iteration test in -short") } @@ -737,17 +720,11 @@ func TestCompareAndSwap_LockPromotedEntry(t *testing.T) { } } -// TestPromote_CASRetryOnConcurrentWrite targets promote's reload-after-failed- -// CAS line (cache.go line 636). The condition: promote holds c.mu, sees -// e.p=nil, tries CAS(nil, c.pd). The CAS fails because a concurrent goroutine -// (Get under c.mu could not run; Swap via unlocked fast path could not, since -// it requires e in read map — wait, it IS in read map here — so this window -// opens when Swap's unlocked CAS loop is active for an entry that was deleted -// during promote's iteration). -// -// We exercise by having many concurrent Swap/Delete/Range cycles so promote -// encounters entries mid-flux. Race-dependent; stress only. -func TestPromote_CASRetryOnConcurrentWrite(t *testing.T) { +// TestSwap_DeleteSwapRangeCycles cycles Delete-then-Swap on a fixed key set +// while Range walks: each cycle drives an entry through tombstone, unlink, +// and re-creation, so Swap's slow path repeatedly encounters entries +// mid-deletion. Race-dependent; stress only. +func TestSwap_DeleteSwapRangeCycles(t *testing.T) { if testing.Short() { t.Skip("skipping stress test in -short") } @@ -784,10 +761,9 @@ func TestPromote_CASRetryOnConcurrentWrite(t *testing.T) { } // TestMemoryBounded churns a fixed key set in and out of the cache. We don't -// assert exact bytes; we assert that after N rounds of (Set, Get, Delete, -// Range-triggered promote), the resident entry count is bounded by the number -// of live keys. This would catch a regression where dirty-map bookkeeping -// leaked entries. +// assert exact bytes; we assert that after N rounds of (Set, Delete, Range), +// the resident entry count is bounded by the number of live keys. This would +// catch a regression where Delete stopped physically unlinking entries. func TestMemoryBounded(t *testing.T) { c := New[int, int]() const liveKeys = 32 @@ -804,7 +780,6 @@ func TestMemoryBounded(t *testing.T) { for i := range liveKeys { c.Set(i, i) } - // Trigger promote. c.Range(func(int, int, error) bool { return true }) } @@ -814,3 +789,87 @@ func TestMemoryBounded(t *testing.T) { t.Fatalf("after %d rounds: %d live entries, want <= %d", rounds, count, liveKeys) } } + +// TestSwap_NotLostDuringConcurrentDelete verifies that a Swap racing a +// Delete of the same key is never silently dropped. The dangerous +// interleaving: Delete tombstones the value slot, Swap observes the +// tombstone, and Delete's physical unlink races Swap's installation of the +// new value. A nil value slot is terminal — Swap must re-create the key as +// a fresh entry, never write into the dying one — so if Swap returns Miss +// ("I stored into nothing"), it linearized after the Delete and the swapped +// value must be visible afterwards. +// +// Regression test: an earlier version resurrected tombstones with a bare +// CAS not serialized with deleteEntryIf's unlink, losing the swapped value +// in ~0.03% of rounds. +func TestSwap_NotLostDuringConcurrentDelete(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const rounds = 100_000 + for i := range rounds { + c := New[int, int]() + c.Set(i, 0) + + var wg sync.WaitGroup + wg.Add(2) + var swapState KeyState + go func() { + defer wg.Done() + c.Delete(i) + }() + go func() { + defer wg.Done() + _, _, swapState = c.Swap(i, 1) + }() + wg.Wait() + + if swapState == Miss { + if v, _, s := c.TryGet(i); s.IsMiss() || v != 1 { + t.Fatalf("round %d: Swap returned Miss (stored after the Delete) but TryGet=(%d,%v); the swapped value was silently dropped", i, v, s) + } + } + } +} + +// TestClean_DoesNotEvictFreshEntries runs Clean continuously against Gets +// on a cache with no MaxAge and no Deletes: nothing ever expires, so Clean +// must never remove anything and every completed Get must be a subsequent +// TryGet hit. +// +// Regression test: an earlier version created Get's entry with a nil value +// slot and CAS'd the loading in afterwards; in that window the fresh entry +// was indistinguishable from a Delete tombstone and Clean would unlink it, +// losing the cached value (and with it, request collapsing for that key). +func TestClean_DoesNotEvictFreshEntries(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + + stop := make(chan struct{}) + var cleanWg sync.WaitGroup + cleanWg.Add(1) + go func() { + defer cleanWg.Done() + for { + select { + case <-stop: + return + default: + c.Clean() + runtime.Gosched() + } + } + }() + + const rounds = 200_000 + for i := range rounds { + c.Get(i, func() (int, error) { return i, nil }) + if v, _, s := c.TryGet(i); s.IsMiss() || v != i { + t.Fatalf("round %d: freshly loaded entry evicted by Clean: TryGet=(%d,%v)", i, v, s) + } + } + close(stop) + cleanWg.Wait() +} diff --git a/cache/trie.go b/cache/trie.go index 168b940..f5b4749 100644 --- a/cache/trie.go +++ b/cache/trie.go @@ -8,8 +8,7 @@ import ( ) // trie is a concurrent hash-trie map, modeled on Go's internal/sync -// HashTrieMap. It is intended to replace Cache's read/dirty backing store. -// For now it is standalone and only exercised by trie_test.go. +// HashTrieMap. It is the backing store for Cache. // // Design notes: // @@ -289,11 +288,13 @@ func (t *trie[K, V]) deleteEntry(k K) *trieEntry[K, V] { } // deleteEntryIf is like deleteEntry, but only proceeds if pred(entry) -// returns true when evaluated under the parent's lock. This lets callers -// guard against racing resurrections: for example, Clean wants to -// physically remove an entry whose value slot is nil, but only if the slot -// is still nil at the moment the lock is acquired (a concurrent Swap may -// have CAS'd a fresh loading in since Clean's initial walk observed nil). +// returns true when evaluated under the parent's lock. Cache uses this to +// physically unlink only tombstoned entries (nil value slot). The value +// slot of a tombstoned entry never transitions back to non-nil (see entDel +// in cache.go), so a nil observed by pred under the lock cannot be +// invalidated by a concurrent writer; and if the key was deleted and +// re-created in the meantime, the lookup finds the fresh live entry and +// pred leaves it alone. // // If pred is nil, deleteEntryIf is equivalent to unconditional deletion. func (t *trie[K, V]) deleteEntryIf(k K, pred func(*trieEntry[K, V]) bool) *trieEntry[K, V] { @@ -335,10 +336,20 @@ func (t *trie[K, V]) deleteEntryIf(k K, pred func(*trieEntry[K, V]) bool) *trieE continue } n = slot.Load() - if n == nil || !n.isEntry { + if n == nil { + // The entry was deleted concurrently; nothing to do. i.mu.Unlock() return nil } + if !n.isEntry { + // The slot expanded into an indirect node between our + // unlocked walk and the lock (a concurrent insert with a + // colliding hash prefix built out this level). The key may + // now live deeper in the new subtree; retry from the root, + // as internal/sync.HashTrieMap's find does. + i.mu.Unlock() + continue + } break } diff --git a/cache/trie_test.go b/cache/trie_test.go index b64a83e..3d8061b 100644 --- a/cache/trie_test.go +++ b/cache/trie_test.go @@ -81,8 +81,9 @@ func TestTrie_LoadOrStoreExisting(t *testing.T) { } func TestTrie_LoadOrStoreWithNilValue(t *testing.T) { - // Cache's Get path constructs an entry with no value yet, then fills it. - // Verify loadOrStoreEntry(k, nil) is legal and the entry.p is nil. + // Cache always creates entries with a value in place (a nil slot means + // tombstoned), but the trie itself is value-agnostic: verify + // loadOrStoreEntry(k, nil) is legal and the entry.p is nil. var tr trie[string, int] e, loaded := tr.loadOrStoreEntry("k", nil) if loaded { @@ -463,8 +464,8 @@ func TestTrie_RangeDuringMutation(t *testing.T) { } // TestTrie_DeleteMissingInCollisionChain walks an overflow chain in search -// of a key that is never present, exercising the "walk to end without -// finding" branch of trieRemoveFromChain. +// of a key that is never present, exercising deleteEntryIf's +// "walk the chain to the end without finding k" branch. func TestTrie_DeleteMissingInCollisionChain(t *testing.T) { var tr trie[string, int] tr.hashFn = func(string) uintptr { return 0x1234 } @@ -529,7 +530,10 @@ func TestTrie_DeleteRaceRetries(t *testing.T) { // TestTrie_DeleteRaceAgainstExpand targets deleteEntryIf's "slot changed to // non-entry after lock" branch. An indirect-node split (expand) during the // window between delete's unlocked walk and its parent.Lock forces the -// reload to see a non-entry node. +// reload to see a non-entry node, which must trigger a retry from the root +// (matching internal/sync.HashTrieMap's find), not a give-up: "a" is present +// for the whole round until the delete removes it, so deleteEntry("a") must +// always find and return it. // // We use a controlled hashFn that collides a few keys in the top bits but // not all bits, so inserting them triggers expand. Concurrent insert+delete @@ -547,7 +551,7 @@ func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { "c": 0xA000000000000002, "d": 0xB000000000000000, } - for range 500 { + for i := range 20_000 { var tr trie[string, int] tr.hashFn = func(k string) uintptr { return hashes[k] } triePut(&tr, "a", 1) @@ -555,6 +559,7 @@ func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { var wg sync.WaitGroup wg.Add(3) + var deleted *trieEntry[string, int] go func() { defer wg.Done() triePut(&tr, "b", 2) @@ -565,9 +570,21 @@ func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { }() go func() { defer wg.Done() - tr.deleteEntry("a") + deleted = tr.deleteEntry("a") }() wg.Wait() + + if deleted == nil || deleted.key != "a" { + t.Fatalf("round %d: deleteEntry(a) = %v, want the entry for a (delete must retry across a concurrent expand)", i, deleted) + } + if e := tr.loadEntry("a"); e != nil { + t.Fatalf("round %d: a still present after deleteEntry returned it", i) + } + for _, k := range []string{"b", "c", "d"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("round %d: %s missing after delete of a", i, k) + } + } } } diff --git a/cache/wrappers_test.go b/cache/wrappers_test.go index 9cdada7..9e5eaa1 100644 --- a/cache/wrappers_test.go +++ b/cache/wrappers_test.go @@ -273,13 +273,29 @@ func TestSet_Clear(t *testing.T) { } } -func TestSet_StopAutoClean(t *testing.T) { +func TestSet_AutoClean(t *testing.T) { + // NewSet must actually start the autoclean goroutine (a regression had + // NewSet building the Cache directly, silently ignoring the option); + // the cleaning behavior itself is covered by the Cache autoclean tests. s := NewSet[string](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + if s.c.quitClean == nil { + t.Fatal("NewSet with AutoCleanInterval should start the autoclean goroutine") + } s.StopAutoClean() // Calling twice is safe (sync.Once on the underlying Cache). s.StopAutoClean() } +func TestItem_AutoClean(t *testing.T) { + i := NewItem[int](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + if i.c.quitClean == nil { + t.Fatal("NewItem with AutoCleanInterval should start the autoclean goroutine") + } + defer i.StopAutoClean() + i.Set(1) + i.Clean() // manual Clean is also exposed on Item +} + func TestSet_ZeroValue(t *testing.T) { var s Set[string] err, ks := s.Get("a", func() error { return nil }) From 05efb829f454755347c520fe5d3d30a9b3aab2e7 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Wed, 10 Jun 2026 12:29:36 -0600 Subject: [PATCH 08/21] cache: make trie test hash constants fit 32-bit uintptr The expand-race test's hash literals overflowed uintptr on 32-bit platforms, so 'go vet' / compiling the tests failed there (caught by a GOARCH=386 cross-vet; the library itself was fine). Build the constants with shifts off ptrBits so the test exercises the same top-4-bits collision shape at either word size. --- cache/trie_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cache/trie_test.go b/cache/trie_test.go index 3d8061b..a4841cd 100644 --- a/cache/trie_test.go +++ b/cache/trie_test.go @@ -544,12 +544,13 @@ func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { } // Keys whose hashes match in the top 4 bits but diverge later: they // share a slot at depth 1 and a leaf at depth 2 (via overflow then - // expand). + // expand). The constants are built by shifting so they fit a uintptr + // on both 64-bit and 32-bit platforms. hashes := map[string]uintptr{ - "a": 0xA000000000000000, - "b": 0xA000000000000001, - "c": 0xA000000000000002, - "d": 0xB000000000000000, + "a": 0xA << (ptrBits - 4), + "b": 0xA<<(ptrBits-4) | 1, + "c": 0xA<<(ptrBits-4) | 2, + "d": 0xB << (ptrBits - 4), } for i := range 20_000 { var tr trie[string, int] From 2f95c76c5dd0753518d750ffd304b5dd540e6e24 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Wed, 10 Jun 2026 13:00:12 -0600 Subject: [PATCH 09/21] cache: harden against audit findings; add CI, linearizability checking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expire could be silently overwritten by a concurrent idle extension: the extension stored a fresh expiry computed from a clock sampled BEFORE loading the expiry, so an Expire whose now()-1 landed between the two samples looked unexpired and was extended over. Extension now (a) loads the expiry before sampling the clock, so any Expire value it observes is necessarily in its past, and (b) CASes against the observed expiry, so an Expire landing between the load and the write wins. Notably, fix (b) alone was not enough — the regression hammer caught the residual (a) window at ~1/2000 rounds. CompareAndSwap/CompareAndDelete now agree with TryGet about what exists: expired entries and errored entries (whose v is just the miss function's placeholder return) no longer match. Get's docs now state the miss function's panic behavior (it runs on an internal goroutine; a panic crashes the process) and the same-key re-entrancy deadlock. New verification layers: - A linearizability checker: batches of concurrent Set/Swap/Delete/ TryGet/CAS/CAD against single keys, verified to be explained by some real-time-consistent sequential ordering. This catches atomicity bugs generically rather than per-interleaving (the prior tombstone- resurrection bug class fails this checker directly). - A trie structural validator (hash placement, overflow chain purity, parent pointers, dead flags, pruning completeness) run at the quiesce point of every concurrent trie test. - GitHub Actions CI: -race on amd64 and arm64 (weaker memory ordering surfaces reordering bugs x86's TSO hides), plus linux/386 without -race (unsupported there) to exercise the 32-bit hash-budget path. The README test-suite paragraph is rewritten to describe the actual layers and the measured 98% coverage. --- .github/workflows/ci.yml | 40 +++++++ README.md | 23 ++-- cache/cache.go | 85 ++++++++++---- cache/cache_test.go | 44 +++++++ cache/concurrency_test.go | 32 +++++ cache/linearizability_test.go | 214 ++++++++++++++++++++++++++++++++++ cache/trie_test.go | 64 ++++++++++ 7 files changed, 475 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 cache/linearizability_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0766e66 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: ci + +on: + push: + branches: ["main"] + pull_request: + +jobs: + test: + # amd64 and arm64: the trie/cache core is lock-free, and arm64's weaker + # memory ordering surfaces reordering bugs that x86's TSO hides. + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: stable + - run: go vet ./... + - run: go test -race ./... + + test-386: + # 32-bit: uintptr hashes are 32 bits, so the trie is 8 levels deep + # instead of 16 and the test hash constants must fit the word size. + # The race detector is unsupported on 386, so this runs without it. + name: test (linux/386) + runs-on: ubuntu-latest + env: + GOARCH: "386" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: stable + - run: go vet ./... + - run: go test ./... diff --git a/README.md b/README.md index 32969c7..099c82a 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,22 @@ value can be kept and returned from `Get` during a refresh / if a refresh fails. Keys can be manually expired with `Expire`. Internally expired values or errors can be occasionally cleaned with `Clean`. -Out of an abundance of paranoia that this code is correct, there are unit -tests targeting ~99% statement coverage of the cache and trie (the remainder -being defensive panics at invariant-violation points and two tightly-timed -race branches that require goroutine interleavings we cannot force from -userspace). All tests against `sync.Map` are copied into this library and -used against `cache.Cache`, and a property test compares the cache's behavior -to a `sync.RWMutex`-guarded `map` oracle over random operation sequences. +Out of an abundance of paranoia that this code is correct, the test suite +layers several kinds of verification: unit tests at ~98% statement coverage +of the cache and trie (the remainder being defensive panics at +invariant-violation points and race-retry branches that require +interleavings that cannot be forced from userspace); all of the stdlib +`sync.Map` tests, copied into this library and run against `cache.Cache`; a +property test comparing the cache to a `sync.RWMutex`-guarded `map` oracle +over random operation sequences; a linearizability checker that runs batches +of concurrent operations against single keys and verifies that a +real-time-consistent sequential ordering explains every observed return; +structural validation of the trie (hash placement, overflow chains, parent +pointers, pruning) after each concurrent stress test; and regression hammers +for specific interleaving bugs found by audit. CI runs the suite under the +race detector on both amd64 and arm64 (whose weaker memory ordering +surfaces reordering bugs that x86 hides), and without it on 32-bit +linux/386. This package also provides a cached `Item` and a `Set`. `Item` can be used to populate an expensive value once and expire or replace it when needed, similar diff --git a/cache/cache.go b/cache/cache.go index 5fdcb89..0cc40ef 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -213,6 +213,12 @@ func initCache[K comparable, V any](c *Cache[K, V], opts ...Opt) { // if the key is not yet cached. If stale values are enabled, the currently // cached value has an error, and there is an unexpired stale value, this // returns the stale value and no error. +// +// The miss function runs on an internal goroutine: if it panics, the process +// crashes (the panic cannot be recovered by the Get caller); recover inside +// miss if you need to survive panics. A miss function must not call Get for +// the same key, in this goroutine or another: the inner Get would wait on +// the load the outer Get is driving and deadlock. func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeyState) { // Fast path: if the entry already exists and holds a live value, use // it without any locking. @@ -530,7 +536,9 @@ func (c *Cache[K, V]) Set(k K, v V) { } // CompareAndSwap swaps the old and new values for k if the value has finished -// loading and the value is equal to old. The type V must be comparable. +// loading without an error, is not expired, and is equal to old. The type V +// must be comparable. Stale values are not considered: only the live value +// is compared against. func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { e := c.t.loadEntry(k) if e == nil { @@ -540,7 +548,9 @@ func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { } // CompareAndDelete deletes the entry for k if the value has finished loading -// and the value is equal to old. The type V must be comparable. +// without an error, is not expired, and is equal to old. The type V must be +// comparable. Stale values are not considered: only the live value is +// compared against. func (c *Cache[K, V]) CompareAndDelete(k K, old V) bool { e := c.t.loadEntry(k) if e == nil { @@ -557,7 +567,7 @@ func (c *Cache[K, V]) CompareAndDelete(k K, old V) bool { func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { l := e.p.Load() - if l == nil || !l.finalized() || any(l.v) != any(old) { + if !casMatches(l, old) { return false } var l2 *loading[V] @@ -569,12 +579,23 @@ func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { return true } l = e.p.Load() - if l == nil || !l.finalized() || any(l.v) != any(old) { + if !casMatches(l, old) { return false } } } +// casMatches reports whether l holds a live cached value equal to old: +// finalized, not errored, not expired. CompareAndSwap and CompareAndDelete +// must agree with TryGet about whether a value exists — an expired or +// errored entry is not a value that can be compared against (an errored +// loading's v is whatever the miss function returned beside the error, +// which was never cached as a value). +func casMatches[V any](l *loading[V], old V) bool { + return l != nil && l.finalized() && l.err == nil && + any(l.v) == any(old) && !l.expired(now()) +} + /////////////////// // ENTRY HELPERS // /////////////////// @@ -705,20 +726,30 @@ func entGet[K comparable, V any](e *ent[K, V], extend time.Duration) (v V, err e // If we waited, we could immediately be expired due to time sync, or // if the user is configured to never cache and they're just using // request collapsing: we still want to return the now expired value. + // + // The expiry must be loaded before the clock is sampled: Expire stores + // now()-1 from its own clock, which can be ahead of a clock sampled + // before our load; sampling after the load guarantees a concurrent + // Expire is classified as expired here rather than extended over. + expires := l.expires.Load() n := now() - if (!waited && l.expired(n)) || l.err != nil { + expired := expires != 0 && expires <= n + if (!waited && expired) || l.err != nil { if l.stale != nil && !l.stale.expired(n) { return l.stale.v, nil, Stale } // The stale value is expired: if our entry is not expired, // this must be an error we waited on. - if !l.expired(n) { + if !expired { return l.v, l.err, Hit } return v, err, state } if extend > 0 && l.err == nil { - l.expires.Store(n + int64(extend)) + // Extend the idle expiry via CAS against the expiry we evaluated + // above so that a concurrent Expire (or Swap finalization) is not + // silently overwritten; if expires changed, the other writer wins. + l.expires.CompareAndSwap(expires, n+int64(extend)) } return l.v, l.err, Hit } @@ -739,22 +770,30 @@ func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, if l.finalized() && l.expires.Load() == 0 && l.err == nil { return l.v, nil, Hit } - if n64 == 0 { - n64 = now() - } - n := n64 - // If we are loading but there is a valid stale, return it, otherwise // return immediately: no get. if !l.finalized() { - if l.stale != nil && !l.stale.expired(n) { + if n64 == 0 { + n64 = now() + } + if l.stale != nil && !l.stale.expired(n64) { return l.stale.v, nil, Stale } return v, err, state } // If we have an error or we are expired, we maybe return the stale. - if expired := l.expired(n); l.err != nil || expired { + // The expiry must be loaded before the clock is sampled (when we are + // the one sampling it); see the matching comment in entGet. A caller- + // provided n64 (Range's batch timestamp) never extends, so the + // ordering does not matter there. + expires := l.expires.Load() + if n64 == 0 { + n64 = now() + } + n := n64 + expired := expires != 0 && expires <= n + if l.err != nil || expired { if l.stale != nil && !l.stale.expired(n) { return l.stale.v, nil, Stale } @@ -763,7 +802,9 @@ func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, } } if extend > 0 && l.err == nil { - l.expires.Store(n + int64(extend)) + // CAS so a concurrent Expire is not silently overwritten; see the + // matching comment in entGet. + l.expires.CompareAndSwap(expires, n+int64(extend)) } return l.v, l.err, Hit } @@ -791,7 +832,8 @@ func NewItem[V any](opts ...Opt) *Item[V] { // Get returns the currently cached value, running the miss function in a // goroutine if the item is not yet cached. If stale values are enabled, the // currently cached value has an error, and there is an unexpired stale value, -// this returns the stale value and no error. +// this returns the stale value and no error. See Cache.Get for the miss +// function's panic and re-entrancy caveats. func (i *Item[V]) Get(miss func() (V, error)) (v V, err error, state KeyState) { return i.c.Get(struct{}{}, miss) } @@ -837,13 +879,15 @@ func (i *Item[V]) Swap(v V) (old V, oldErr error, oldState KeyState) { } // CompareAndSwap swaps the old and new values if the value has finished -// loading and the value is equal to old. The type V must be comparable. +// loading without an error, is not expired, and is equal to old. The type V +// must be comparable. func (i *Item[V]) CompareAndSwap(old, new V) (swapped bool) { return i.c.CompareAndSwap(struct{}{}, old, new) } -// CompareAndDelete deletes the item if the value has finished loading and the -// value is equal to old. The type V must be comparable. +// CompareAndDelete deletes the item if the value has finished loading +// without an error, is not expired, and is equal to old. The type V must be +// comparable. func (i *Item[V]) CompareAndDelete(old V) (deleted bool) { return i.c.CompareAndDelete(struct{}{}, old) } @@ -890,7 +934,8 @@ func NewSet[K comparable](opts ...Opt) *Set[K] { // Get ensures the key is cached, running the miss function in a goroutine if // the key is not yet cached. If stale keys are enabled, the currently cached // key has an error, and there is a stale key, this returns with no error and a -// Stale key state. +// Stale key state. See Cache.Get for the miss function's panic and +// re-entrancy caveats. func (s *Set[K]) Get(k K, miss func() error) (err error, state KeyState) { _, err, state = s.c.Get(k, func() (struct{}, error) { return struct{}{}, miss() diff --git a/cache/cache_test.go b/cache/cache_test.go index 4827063..f3f9d56 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -444,3 +444,47 @@ func TestMaxIdleAge(t *testing.T) { vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) } + +// TestCompareAndSwapAgreesWithTryGet verifies that CompareAndSwap and +// CompareAndDelete only match live values: expired entries and errored +// entries are "not there" per TryGet, so comparing against them must fail. +func TestCompareAndSwapAgreesWithTryGet(t *testing.T) { + t.Run("expired", func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + c.Expire("k") + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatal("expired entry should TryGet Miss") + } + if c.CompareAndSwap("k", 1, 2) { + t.Error("CompareAndSwap matched an expired value") + } + if c.CompareAndDelete("k", 1) { + t.Error("CompareAndDelete matched an expired value") + } + }) + + t.Run("errored", func(t *testing.T) { + c := New[string, int]() + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + // The errored loading's v is the zero int; CAS against 0 must not + // treat that as a cached value. + if c.CompareAndSwap("k", 0, 2) { + t.Error("CompareAndSwap matched an errored entry's placeholder value") + } + if c.CompareAndDelete("k", 0) { + t.Error("CompareAndDelete matched an errored entry's placeholder value") + } + }) + + t.Run("live", func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + if !c.CompareAndSwap("k", 1, 2) { + t.Error("CompareAndSwap failed on a live matching value") + } + if !c.CompareAndDelete("k", 2) { + t.Error("CompareAndDelete failed on a live matching value") + } + }) +} diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index 5417c45..ccf2950 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -873,3 +873,35 @@ func TestClean_DoesNotEvictFreshEntries(t *testing.T) { close(stop) cleanWg.Wait() } + +// TestExpire_NotLostToIdleExtension verifies that Expire racing an +// idle-extending read sticks: once both return, the key must be expired. +// +// Regression test: idle extension previously used a plain Store on the +// expiry, which could overwrite a concurrent Expire (read loads expiry, +// Expire stores now-1, read stores now+idle). Extension now CASes against +// the expiry it observed, so an interleaved Expire wins. +func TestExpire_NotLostToIdleExtension(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int](MaxIdleAge(time.Hour)) + const rounds = 100_000 + for i := range rounds { + c.Set(i, 1) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + c.TryGet(i) // a Hit extends the idle expiry + }() + go func() { + defer wg.Done() + c.Expire(i) + }() + wg.Wait() + if _, _, s := c.TryGet(i); s.IsHit() { + t.Fatalf("round %d: Expire was overwritten by a concurrent idle extension", i) + } + } +} diff --git a/cache/linearizability_test.go b/cache/linearizability_test.go new file mode 100644 index 0000000..493abee --- /dev/null +++ b/cache/linearizability_test.go @@ -0,0 +1,214 @@ +package cache + +import ( + "fmt" + "math/rand/v2" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// This file implements a small linearizability checker: each round runs a +// handful of concurrent map operations against a single key, records every +// operation's invocation/response order and return values, and then +// verifies that SOME linearization exists — a total order of the operations +// consistent with real time (op A precedes op B if A responded before B was +// invoked) under which sequential map semantics produce exactly the +// observed returns. If no such order exists, an atomicity bug has been +// observed and the full history is dumped. +// +// Get is deliberately excluded from histories: its singleflight contract +// makes it non-atomic by design (a concurrent Swap cancels an in-flight +// load and the Get returns the Swap's value with a Miss state), which is +// documented behavior but inexpressible as a single sequential operation. +// The cache is configured with no expiry options, so the model is exactly +// a map. + +type linKind uint8 + +const ( + linSet linKind = iota + linSwap + linDelete + linTryGet + linCAS + linCAD +) + +func (k linKind) String() string { + return [...]string{"Set", "Swap", "Delete", "TryGet", "CAS", "CAD"}[k] +} + +type linOp struct { + kind linKind + arg int // value for Set/Swap, old for CAS/CAD + arg2 int // new for CAS + + retV int + retS KeyState + retB bool + + start, end int64 // global ticket counter values around the call +} + +func (o *linOp) String() string { + switch o.kind { + case linSet: + return fmt.Sprintf("Set(%d) [%d,%d]", o.arg, o.start, o.end) + case linSwap: + return fmt.Sprintf("Swap(%d)=(%d,%v) [%d,%d]", o.arg, o.retV, o.retS, o.start, o.end) + case linDelete: + return fmt.Sprintf("Delete=(%d,%v) [%d,%d]", o.retV, o.retS, o.start, o.end) + case linTryGet: + return fmt.Sprintf("TryGet=(%d,%v) [%d,%d]", o.retV, o.retS, o.start, o.end) + case linCAS: + return fmt.Sprintf("CAS(%d,%d)=%v [%d,%d]", o.arg, o.arg2, o.retB, o.start, o.end) + case linCAD: + return fmt.Sprintf("CAD(%d)=%v [%d,%d]", o.arg, o.retB, o.start, o.end) + } + return "?" +} + +type linState struct { + present bool + val int +} + +// linApply runs op against the sequential model state, reporting the next +// state and whether the operation's recorded returns are consistent with +// running at this point in the order. +func linApply(st linState, o *linOp) (linState, bool) { + switch o.kind { + case linSet: + return linState{true, o.arg}, true + case linSwap: + if st.present { + return linState{true, o.arg}, o.retS == Hit && o.retV == st.val + } + return linState{true, o.arg}, o.retS == Miss + case linDelete: + if st.present { + return linState{}, o.retS == Hit && o.retV == st.val + } + return linState{}, o.retS == Miss + case linTryGet: + if st.present { + return st, o.retS == Hit && o.retV == st.val + } + return st, o.retS == Miss + case linCAS: + if st.present && st.val == o.arg { + return linState{true, o.arg2}, o.retB + } + return st, !o.retB + case linCAD: + if st.present && st.val == o.arg { + return linState{}, o.retB + } + return st, !o.retB + } + panic("unreachable") +} + +// linearizable reports whether some real-time-consistent total order of ops +// explains every op's returns starting from initial. +func linearizable(initial linState, ops []*linOp) bool { + used := make([]bool, len(ops)) + var rec func(st linState, depth int) bool + rec = func(st linState, depth int) bool { + if depth == len(ops) { + return true + } + next: + for i, o := range ops { + if used[i] { + continue + } + // o may go next only if no other unscheduled op wholly + // preceded it in real time. + for j, p := range ops { + if !used[j] && j != i && p.end < o.start { + continue next + } + } + st2, ok := linApply(st, o) + if !ok { + continue + } + used[i] = true + if rec(st2, depth+1) { + return true + } + used[i] = false + } + return false + } + return rec(initial, 0) +} + +func TestLinearizability(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const ( + rounds = 50_000 + opsPer = 4 + valSpace = 3 // small domain so CAS/CAD old values plausibly match + ) + c := New[int, int]() // shared across rounds so the trie grows/expands/prunes + + var ticket atomic.Int64 + for round := range rounds { + k := round + initial := linState{} + if rand.IntN(2) == 1 { + initial = linState{true, 1 + rand.IntN(valSpace)} + c.Set(k, initial.val) // sequentially primed before the round + } + + ops := make([]*linOp, opsPer) + for i := range ops { + ops[i] = &linOp{ + kind: linKind(rand.IntN(6)), + arg: 1 + rand.IntN(valSpace), + arg2: 1 + rand.IntN(valSpace), + } + } + + var wg sync.WaitGroup + wg.Add(len(ops)) + for _, o := range ops { + go func(o *linOp) { + defer wg.Done() + o.start = ticket.Add(1) + switch o.kind { + case linSet: + c.Set(k, o.arg) + case linSwap: + o.retV, _, o.retS = c.Swap(k, o.arg) + case linDelete: + o.retV, _, o.retS = c.Delete(k) + case linTryGet: + o.retV, _, o.retS = c.TryGet(k) + case linCAS: + o.retB = c.CompareAndSwap(k, o.arg, o.arg2) + case linCAD: + o.retB = c.CompareAndDelete(k, o.arg) + } + o.end = ticket.Add(1) + }(o) + } + wg.Wait() + + if !linearizable(initial, ops) { + var dump strings.Builder + for _, o := range ops { + dump.WriteString("\n\t") + dump.WriteString(o.String()) + } + t.Fatalf("round %d: no linearization explains this history (initial present=%v val=%d):%s", + round, initial.present, initial.val, dump.String()) + } + } +} diff --git a/cache/trie_test.go b/cache/trie_test.go index a4841cd..b0e374c 100644 --- a/cache/trie_test.go +++ b/cache/trie_test.go @@ -29,6 +29,62 @@ func trieGet[K comparable, V any](t *trie[K, V], k K) (V, bool) { return *p, true } +// verifyTrie checks structural invariants once concurrent operations have +// quiesced: every entry hangs at the position prescribed by its hash, +// overflow chains hold only full-hash collisions, parent pointers are +// consistent, no linked node is marked dead, and no non-root indirect is +// empty (pruning must have removed it). It must not run concurrently with +// mutations. +func verifyTrie[K comparable, V any](t *testing.T, tr *trie[K, V]) { + t.Helper() + if !tr.inited.Load() { + return + } + root := tr.root.Load() + var walk func(i *trieIndirect[K, V], prefix uintptr, nbits uint) + walk = func(i *trieIndirect[K, V], prefix uintptr, nbits uint) { + if i.dead.Load() { + t.Fatalf("linked indirect at depth %d is marked dead", nbits/trieBranchingLog2) + } + if i != root && i.empty() { + t.Fatalf("empty non-root indirect linked at depth %d (pruning missed it)", nbits/trieBranchingLog2) + } + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + cp := prefix< ptrBits { + t.Fatal("trie deeper than the hash bit budget") + } + if n.isEntry { + head := n.entry() + headHash := tr.hash(head.key) + for e := head; e != nil; e = e.overflow.Load() { + h := tr.hash(e.key) + if h>>(ptrBits-cb) != cp { + t.Fatalf("entry for key %v misplaced: hash %#x does not match its path %#x at depth %d", + e.key, h, cp, cb/trieBranchingLog2) + } + if h != headHash { + t.Fatalf("overflow chain mixes hashes: key %v hash %#x vs head %v hash %#x", + e.key, h, head.key, headHash) + } + } + } else { + ci := n.indirect() + if ci.parent != i { + t.Fatalf("indirect at depth %d has a wrong parent pointer", cb/trieBranchingLog2) + } + walk(ci, cp, cb) + } + } + } + walk(root, 0, 0) +} + func TestTrie_ZeroValueLoad(t *testing.T) { var tr trie[string, int] if e := tr.loadEntry("missing"); e != nil { @@ -237,6 +293,7 @@ func TestTrie_ManyKeys(t *testing.T) { } } } + verifyTrie(t, &tr) } // TestTrie_FullHashCollision verifies the overflow-chain behavior by forcing @@ -340,6 +397,7 @@ func TestTrie_ConcurrentStoreLoad(t *testing.T) { }(w) } wg.Wait() + verifyTrie(t, &tr) } // TestTrie_ConcurrentSameKey hammers the same key with many goroutines @@ -379,6 +437,7 @@ func TestTrie_ConcurrentSameKey(t *testing.T) { t.Fatalf("iter %d: worker %d saw entry %p, want %p", iter, i, e, first) } } + verifyTrie(t, &tr) } } @@ -411,6 +470,7 @@ func TestTrie_ConcurrentDeletePrune(t *testing.T) { if root := tr.root.Load(); !root.empty() { t.Fatal("root not pruned: has surviving children") } + verifyTrie(t, &tr) } // TestTrie_RangeDuringMutation runs walk concurrently with store and @@ -461,6 +521,7 @@ func TestTrie_RangeDuringMutation(t *testing.T) { } stop.Store(true) wg.Wait() + verifyTrie(t, &tr) } // TestTrie_DeleteMissingInCollisionChain walks an overflow chain in search @@ -524,6 +585,7 @@ func TestTrie_DeleteRaceRetries(t *testing.T) { }(w) } wg.Wait() + verifyTrie(t, &tr) } } @@ -586,6 +648,7 @@ func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { t.Fatalf("round %d: %s missing after delete of a", i, k) } } + verifyTrie(t, &tr) } } @@ -614,4 +677,5 @@ func TestTrie_DeleteDuringConcurrentStore(t *testing.T) { } stop.Store(true) wg.Wait() + verifyTrie(t, &tr) } From be44fedd2e0c89ccc6a35a37c093f3466c194413 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Wed, 10 Jun 2026 20:12:21 -0600 Subject: [PATCH 10/21] cache: monotonic clock, saturating expiry math, race-free idle extension Implements the second audit round (independent session), verified by a review pass with full interleaving traces and mutation-tested regressions: - now() is monotonic (time.Since of a process epoch, padded 1s clear of the 0 sentinel). This makes the load-expiry-then-sample-clock ordering argument airtight rather than dependent on wall-clock sanity, and immunizes TTLs against NTP steps. Trade-off (suspend pauses aging) documented on epoch. - Idle extension can no longer resurrect expired values: the extension CAS is gated on !expired, fixing the waited-expired path where a MaxAge(0) collapse value was revived for the idle window; and the driving Get's tail no longer extends (it publicly returns Miss, not a Hit, and extension was silently rewriting the MaxAge-born TTL). - Clean and idle extension now contend on the expiry word: Clean claims it (CAS to the expired sentinel) before tombstoning, so a racing extension either wins (entry kept) or loses the same way it loses to Expire, instead of being silently thrown away by the eviction. - Clean no longer evicts entries whose stale is still being served: for born-expired entries (MaxAge(0)/MaxErrorAge(0) store -1) the grace window anchored at the epoch rather than the entry. Pre-existing bug found while verifying the Clean change. - Get's replace decision is re-derived from prev itself, and the stale piggyback is pinned to the slot it read, so a load that finalizes live in the window between Get's slot read and entGet's cannot be demoted to a stale and redundantly re-driven. - Delete and Swap classify the displaced value with a clock sampled at the displacing CAS, not after, so a value removed live is not misreported as expired. - All expiry arithmetic saturates at MaxInt64 (satAdd), so huge Durations mean "effectively forever" instead of wrapping negative into born-expired (or, in Clean, evict-everything). - Doc fixes: NaN / non-comparable key behavior, MaxErrorAge's no-throttling-with-stales caveat, MaxIdleAge best-effort note, MaxStaleAge without MaxAge, autoclean pinning the cache, terminal- tombstone contract on trieEntry, StopAutoClean/Set wording. All four new regression tests fail against the reverted implementation (the redrive test by double-close panic); the full -race suite, the linearizability checker, and the prior round's hammers pass on amd64, with vet clean on linux/386. --- cache/cache.go | 174 +++++++++++++++++++++++++++++--------- cache/cache_test.go | 74 ++++++++++++++++ cache/concurrency_test.go | 50 +++++++++++ cache/trie.go | 9 +- 4 files changed, 264 insertions(+), 43 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 0cc40ef..d55ce27 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -17,6 +17,7 @@ package cache import ( + "math" "sync" "sync/atomic" "time" @@ -63,6 +64,13 @@ type ( // Cache caches comparable keys to arbitrary values. By default, the // cache grows without bounds and all keys persist forever. These // limits can be changed with options that are passed to New. + // + // Keys behave like the builtin map's: a key containing a NaN is + // never equal to itself, so it can be stored but never found, + // deleted, or cleaned again — every Get with such a key runs the + // miss function and adds an entry that only Clear removes. Likewise, + // an interface key holding a non-comparable dynamic value panics, + // as with a map. Cache[K comparable, V any] struct { t trie[K, loading[V]] @@ -100,7 +108,29 @@ const ( // trie stays a generic hash-trie with no cache-specific logic. type ent[K comparable, V any] = trieEntry[K, loading[V]] -func now() int64 { return time.Now().UnixNano() } +// epoch anchors now. Expiry math uses the monotonic clock so that two +// causally ordered samples can never run backwards: entGet's "load the +// expiry, then sample the clock" argument — and Expire reliably winning +// races against idle extension — relies on that, and a wall-clock step +// (NTP, manual adjustment) would otherwise move every TTL. The one-second +// pad keeps now strictly positive so it can never collide with the 0 +// sentinels ("no expiry" / "caller samples the clock"). Trade-off: on +// platforms whose monotonic clock pauses during system suspend, entries +// age only while the system runs. +var epoch = time.Now().Add(-time.Second) + +func now() int64 { return int64(time.Since(epoch)) } + +// satAdd returns a+b saturated at math.MaxInt64, for expiry arithmetic on +// nanosecond quantities; b must be non-negative (a may be the -1 expired +// sentinel). Without saturation, a huge MaxAge / MaxStaleAge / MaxIdleAge +// would wrap negative and make entries born (or instantly) expired. +func satAdd(a, b int64) int64 { + if s := a + b; s >= a { + return s + } + return math.MaxInt64 +} func (cfg *cfg) newExpires(err error) int64 { var ttl time.Duration @@ -121,7 +151,7 @@ func (cfg *cfg) newExpires(err error) int64 { if ttl == 0 { return 0 } - return time.Now().Add(ttl).UnixNano() + return satAdd(now(), int64(ttl)) } func (o opt) apply(c *cfg) { o.fn(c) } @@ -143,8 +173,8 @@ func MaxAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxAge, c.a // after an entry has expired (so, total age is MaxAge + MaxStaleAge). A stale // value is the previous successfully cached value that is returned while the // value is being refreshed (a new value is being queried). As well, the stale -// value is returned while the refreshed value is erroring. This option is -// useless without MaxAge. +// value is returned while the refreshed value is erroring. Without MaxAge, +// stales still apply to manually Expired keys and to erroring refreshes. // // A special value of -1 allows stale values to be returned indefinitely. func MaxStaleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxStaleAge = age }} } @@ -152,14 +182,23 @@ func MaxStaleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxSta // MaxErrorAge sets the age to persist load errors. If not specified, the // default is MaxAge. Using this option with 0 disables caching errors // entirely. +// +// A cached error suppresses repeat queries only when there is no stale +// value to return: if stale values are enabled and a valid stale exists, +// Get returns the stale and always drives a refresh, regardless of the +// error's remaining age. func MaxErrorAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxErrAge, c.errAgeSet = age, true }} } // MaxIdleAge opts in to extending an entry's expiry on each successful -// access. Each time Get or TryGet returns a Hit without an error, the +// access. Each time Get or TryGet returns a live Hit without an error, the // entry's expiry is reset to now + age. If MaxAge is not set, the idle // age is also used as the initial TTL. +// +// The reset is best effort under races: a concurrent Expire, or a Clean +// that has already deemed the entry expired, wins over an in-flight +// extension. func MaxIdleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxIdleAge = age }} } @@ -169,6 +208,9 @@ func MaxIdleAge(age time.Duration) Opt { // interval that is less than your MaxAge + MaxStaleAge. Values are only // candidates to be cleaned after the max age has elapsed. At worst, a value // may persist for a total of MaxAge + MaxStaleAge + AutoCleanInterval. +// +// The goroutine holds a reference to the cache, so a cache that started +// autocleaning is not garbage collected until StopAutoClean is called. func AutoCleanInterval(interval time.Duration) Opt { return opt{fn: func(c *cfg) { c.autoCleanInterval = interval }} } @@ -263,11 +305,25 @@ outer: if v, err, s = entGet(e, c.cfg.maxIdleAge); s == Hit { return v, err, s } - if s == Stale && !prev.finalized() { + if s == Stale && !prev.finalized() && e.p.Load() == prev { // A load is already in flight and its caller will - // finalize it; piggyback and return the stale now. + // finalize it; piggyback and return the stale now. The + // slot re-check pins the stale to prev: entGet loads + // the slot itself, so without it the stale could come + // from a newer loading while the in-flight test + // inspects prev. return v, err, s } + // Replace prev only if it is finalized and expired or + // errored. entGet's non-Hit result usually implies that, + // but entGet re-loads the slot itself: a racing writer can + // finalize prev live (or replace it) between our load of + // prev and entGet's. Re-deriving from prev keeps us from + // demoting a freshly finalized live value to a stale and + // driving a redundant load; the retry re-reads the slot. + if !prev.finalized() || (prev.err == nil && !prev.expired(now())) { + continue + } // prev is finalized and expired or errored, possibly with a // valid stale that nothing is refreshing (if it was in // flight with no stale, entGet waited for it to finalize). @@ -289,8 +345,11 @@ outer: // Wait for our loading to finalize, then return the value. Waiting via // l.wg (our own) synchronizes with whoever called wg.Done — the miss // goroutine's setve or a racing Swap's defer — even if e.p has since - // been replaced by another writer. - v, err, s = entGet(e, c.cfg.maxIdleAge) + // been replaced by another writer. extend=0: this Get drove the load + // and publicly returns Miss, so it is not an idle-extending access + // (MaxIdleAge documents extension on Hits); the value keeps the + // initial TTL it was born with. + v, err, s = entGet(e, 0) switch s { case Miss, Hit: l.wg.Wait() @@ -330,12 +389,18 @@ func (c *Cache[K, V]) Delete(k K) (v V, err error, _ KeyState) { // (see entDel), so entDead observed under the bucket lock is stable; a // concurrent Swap or Get that finds the tombstone re-creates the key // as a fresh entry, which the predicate leaves alone. + // + // The clock is sampled before the tombstone CAS so the removed value + // is classified (live, stale, or expired) as of the removal itself; + // sampling after would let a value that expires in between be + // misreported as already gone even though this Delete removed it live. + n := now() was := entDel(e) c.t.deleteEntryIf(k, entDead[K, V]) if was == nil { return v, err, Miss } - return loadingTryGet(was, 0, 0) + return loadingTryGet(was, n, 0) } // Expire sets a stored value to expire immediately, meaning the next Get will @@ -372,7 +437,9 @@ func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // Clean deletes all expired values from the cache. A value is expired if // MaxAge is used and the entry is older than the max age, or if you manually // expired a key. If MaxStaleAge is used and not -1, the entry must be older -// than MaxAge + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. +// than MaxAge + MaxStaleAge, and an entry whose stale value is still within +// its window is never removed. If MaxStaleAge is -1, Clean returns +// immediately. // // Clean also physically removes entries that were tombstoned by prior Delete // calls, so the trie's memory footprint does not grow unboundedly with @@ -396,8 +463,22 @@ func (c *Cache[K, V]) Clean() { return true } expires := l.expires.Load() - if expires != 0 && tn > expires+int64(c.cfg.maxStaleAge) { - if e.p.CompareAndSwap(l, nil) { + // The stale check matters for born-expired entries (MaxAge(0) / + // MaxErrorAge(0) store the -1 sentinel): their grace window would + // otherwise anchor at the epoch rather than at the entry, evicting + // stales that TryGet would still serve. For normally expired + // entries the stale dies at expires+maxStaleAge anyway, so the + // check changes nothing. + if expires != 0 && tn > satAdd(expires, int64(c.cfg.maxStaleAge)) && + (l.stale == nil || l.stale.expired(tn)) { + // Win the expiry word before tombstoning: idle extension + // CASes against the expiry it observed, so claiming the + // word here (writing the expired sentinel) makes a racing + // extension fail — the same way it loses to Expire — rather + // than be silently thrown away by our eviction. If instead + // the extension wins, the entry is in use; leave it for a + // later pass. + if l.expires.CompareAndSwap(expires, -1) && e.p.CompareAndSwap(l, nil) { toPrune = append(toPrune, e.key) } } @@ -419,8 +500,8 @@ func (c *Cache[K, V]) Clear() { c.t.clear() } -// StopAutoClean stops the auto clean goroutine that is began with the -// AutoClean option. +// StopAutoClean stops the auto clean goroutine that is begun with the +// AutoCleanInterval option. func (c *Cache[K, V]) StopAutoClean() { c.quitOnce.Do(func() { if c.quitClean != nil { @@ -438,14 +519,19 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { l := c.finalizedLoading(v) var was *loading[V] + var wasN int64 defer func() { if was == nil { return } // The following block is similar to setve, but expanded to - // return our prior value (or stale, or err) if it exists. - n := now() + // return our prior value (or stale, or err) if it exists. wasN + // was sampled just before the CAS that displaced was, so the + // displaced value is classified (live, stale, or expired) as of + // the swap itself; a fresh sample here could misreport a value + // that expired between the CAS and this defer. + n := wasN if !was.finalized() { was.mu.Lock() if !was.finalized() { @@ -479,7 +565,7 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { // re-create the key rather than writing into the dead entry. if e := c.t.loadEntry(k); e != nil { if rm := e.p.Load(); rm != nil { - was = rm + was, wasN = rm, now() if e.p.CompareAndSwap(rm, l) { return old, oldErr, oldState } @@ -506,7 +592,7 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { c.t.deleteEntryIf(k, entDead[K, V]) break } - was = rm + was, wasN = rm, now() if e.p.CompareAndSwap(rm, l) { return old, oldErr, oldState } @@ -699,12 +785,14 @@ func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { if expires <= 0 { expires = now() } - return &stale[V]{v, expires + int64(age)} + return &stale[V]{v, satAdd(expires, int64(age))} } -// entGet always returns the value or the stale value. We do not check if our -// value is expired: we call this at the end of Get, we must always return -// something even if it is to be immediately expired. +// entGet returns the value, the stale value, or — after waiting on an +// in-flight load — whatever the load produced. When we waited, expiry alone +// does not force a miss: Get must hand its caller something even if the +// value expired the instant it loaded (e.g. request collapsing with +// MaxAge(0)). func entGet[K comparable, V any](e *ent[K, V], extend time.Duration) (v V, err error, state KeyState) { l := entLoad(e) var waited bool @@ -745,11 +833,15 @@ func entGet[K comparable, V any](e *ent[K, V], extend time.Duration) (v V, err e } return v, err, state } - if extend > 0 && l.err == nil { + if extend > 0 && l.err == nil && !expired { // Extend the idle expiry via CAS against the expiry we evaluated - // above so that a concurrent Expire (or Swap finalization) is not - // silently overwritten; if expires changed, the other writer wins. - l.expires.CompareAndSwap(expires, n+int64(extend)) + // above so that a concurrent Expire (or Swap finalization, or a + // Clean eviction) is not silently overwritten; if expires + // changed, the other writer wins. Never extend an expired entry: + // a waited-on load that finalized already expired (MaxAge(0) + // collapsing, or a TTL shorter than the load itself) is returned + // as a courtesy, not resurrected. + l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) } return l.v, l.err, Hit } @@ -785,8 +877,8 @@ func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, // If we have an error or we are expired, we maybe return the stale. // The expiry must be loaded before the clock is sampled (when we are // the one sampling it); see the matching comment in entGet. A caller- - // provided n64 (Range's batch timestamp) never extends, so the - // ordering does not matter there. + // provided n64 (Range's batch timestamp, Delete's pre-removal clock) + // never extends, so the ordering does not matter there. expires := l.expires.Load() if n64 == 0 { n64 = now() @@ -803,8 +895,10 @@ func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, } if extend > 0 && l.err == nil { // CAS so a concurrent Expire is not silently overwritten; see the - // matching comment in entGet. - l.expires.CompareAndSwap(expires, n+int64(extend)) + // matching comment in entGet. This path is unreachable when + // expired (the branch above returns), so unlike entGet no + // explicit !expired gate is needed. + l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) } return l.v, l.err, Hit } @@ -905,8 +999,8 @@ func (i *Item[V]) Clean() { i.c.Clean() } -// StopAutoClean stops the auto clean goroutine that is began with the -// AutoClean option. +// StopAutoClean stops the auto clean goroutine that is begun with the +// AutoCleanInterval option. func (i *Item[V]) StopAutoClean() { i.c.StopAutoClean() } @@ -923,7 +1017,7 @@ type Set[K comparable] struct { } // NewSet returns a new Set, with the optional overrides configuring cache -// semantics. If you do not need to configure an set at all, the zero value set +// semantics. If you do not need to configure a set at all, the zero value set // is valid and usable. func NewSet[K comparable](opts ...Opt) *Set[K] { s := new(Set[K]) @@ -943,10 +1037,10 @@ func (s *Set[K]) Get(k K, miss func() error) (err error, state KeyState) { return err, state } -// TryGet any error for the given key if it is cached. This returns either the -// currently stored nil error, or if the current store has an error, the stale -// nil error if present, otherwise the current error. If nothing is cached, or -// what is cached is expired, this returns Miss. +// TryGet returns the cached state for the given key: nil if the key loaded +// successfully, a stale nil if the latest load errored but a valid stale +// exists, or the load error itself. If nothing is cached, or what is cached +// is expired, this returns Miss. func (s *Set[K]) TryGet(k K) (err error, state KeyState) { _, err, state = s.c.TryGet(k) return err, state @@ -996,8 +1090,8 @@ func (s *Set[K]) Clear() { s.c.Clear() } -// StopAutoClean stops the auto clean goroutine that is began with the -// AutoClean option. +// StopAutoClean stops the auto clean goroutine that is begun with the +// AutoCleanInterval option. func (s *Set[K]) StopAutoClean() { s.c.StopAutoClean() } diff --git a/cache/cache_test.go b/cache/cache_test.go index f3f9d56..5330ee5 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -2,6 +2,7 @@ package cache import ( "errors" + "math" "sync" "sync/atomic" "testing" @@ -426,6 +427,20 @@ func TestMaxIdleAge(t *testing.T) { vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) + // MaxIdleAge must not resurrect entries that finalize already expired: + // with MaxAge(0), caching is disabled entirely and the Get that drives + // (or piggybacks on) the load gets a courtesy value, not an + // idle-extending access. Regression test: the extension previously + // fired on the waited-expired path and revived the value for the idle + // window. + t.Run("no_resurrect_expired", func(t *testing.T) { + c := New[string, string](MaxAge(0), MaxIdleAge(time.Hour)) + v, err, s := c.Get("foo", func() (string, error) { return "bar", nil }) + vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Miss}) + v, err, s = c.TryGet("foo") + vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) + }) + // Range doesn't extend. t.Run("range_no_extend", func(t *testing.T) { const ttl = 200 * time.Millisecond @@ -445,6 +460,65 @@ func TestMaxIdleAge(t *testing.T) { }) } +// TestHugeAges verifies expiry arithmetic saturates rather than wrapping: +// absurd-but-valid Durations must mean "effectively forever", not "born +// expired" (or, for Clean, "evict everything"). +func TestHugeAges(t *testing.T) { + huge := time.Duration(math.MaxInt64) + + // MaxAge: now + ttl must not wrap negative. + { + c := New[string, int](MaxAge(huge)) + c.Set("k", 1) + if v, _, s := c.TryGet("k"); !s.IsHit() || v != 1 { + t.Fatalf("TryGet under huge MaxAge: v=%d s=%v, want 1 Hit", v, s) + } + } + + // MaxStaleAge: Clean's expires + staleAge must not wrap negative, and + // the stale's own expiry must not be born wrapped. + { + c := New[string, int](MaxAge(time.Nanosecond), MaxStaleAge(huge)) + c.Set("k", 1) + time.Sleep(time.Millisecond) // main entry expired, stale effectively forever + c.Clean() // must not evict + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet after Clean under huge MaxStaleAge: v=%d s=%v, want 1 Stale", v, s) + } + } + + // MaxIdleAge: the extension n + idle must not wrap negative. + { + c := New[string, int](MaxIdleAge(huge)) + c.Set("k", 1) + c.TryGet("k") // a Hit extends; the extension must saturate + if v, _, s := c.TryGet("k"); !s.IsHit() || v != 1 { + t.Fatalf("TryGet after huge idle extension: v=%d s=%v, want 1 Hit", v, s) + } + } +} + +// TestClean_RespectsValidStales verifies Clean never removes an entry whose +// stale is still within its window. Born-expired entries (MaxAge(0) stores +// the -1 expired sentinel) are the regression case: their MaxAge+MaxStaleAge +// grace would otherwise anchor at the epoch instead of at the entry, so +// Clean evicted stales that TryGet was still serving. +func TestClean_RespectsValidStales(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(50*time.Millisecond)) + c.Set("k", 1) + + c.Clean() // stale still valid: must not evict + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet after early Clean: v=%d s=%v, want 1 Stale", v, s) + } + + time.Sleep(60 * time.Millisecond) // stale now expired + c.Clean() // must evict + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after late Clean: s=%v, want Miss", s) + } +} + // TestCompareAndSwapAgreesWithTryGet verifies that CompareAndSwap and // CompareAndDelete only match live values: expired entries and errored // entries are "not there" per TryGet, so comparing against them must fail. diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index ccf2950..0558ab3 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -874,6 +874,56 @@ func TestClean_DoesNotEvictFreshEntries(t *testing.T) { cleanWg.Wait() } +// TestGet_DoesNotRedriveFreshlyFinalizedValue targets the slow-path window +// between Get's load of prev and entGet's own load of the slot: if the +// in-flight load it observed finalizes live in that window, Get must return +// the Hit rather than demote the fresh value to a stale and drive a +// redundant load. Every round arranges an expired-with-stale entry, races +// Gets against the first load's release, and asserts the miss function ran +// exactly once. +func TestGet_DoesNotRedriveFreshlyFinalizedValue(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 2000 { + c := New[int, int](MaxAge(time.Hour), MaxStaleAge(time.Hour)) + c.Set(iter, 1) + c.Expire(iter) // expired with a valid self-stale + + // A driving Get returns the stale and runs its miss on a + // background goroutine, so completion must be synchronized on: + // missDone is closed by the (single) miss, and a redundant + // second drive panics on the double close. + release := make(chan struct{}) + missDone := make(chan struct{}) + miss := func() (int, error) { + <-release // closed below; late drivers pass straight through + close(missDone) + return 2, nil + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + c.Get(iter, miss) // drives the refresh, returns Stale(1) + }() + go func() { + defer wg.Done() + runtime.Gosched() + close(release) + }() + // Meanwhile, hammer Gets that race the finalization. None may + // drive a second load: each must piggyback the stale or return + // the fresh Hit. + for range 4 { + c.Get(iter, miss) + } + wg.Wait() + <-missDone + } +} + // TestExpire_NotLostToIdleExtension verifies that Expire racing an // idle-extending read sticks: once both return, the key must be expired. // diff --git a/cache/trie.go b/cache/trie.go index f5b4749..0295911 100644 --- a/cache/trie.go +++ b/cache/trie.go @@ -61,9 +61,12 @@ type trieIndirect[K comparable, V any] struct { } // trieEntry is a leaf node. The value slot p is an atomic pointer; a nil -// value slot means the entry is tombstoned but still wired into the trie -// (callers can atomically replace nil with a fresh value). The overflow -// chain handles hash-equal keys (full hash collisions). +// value slot means the entry is tombstoned but still wired into the trie. +// The cache treats a nil slot as terminal — it never stores a value over +// one, and re-creates the key as a fresh entry instead (see entDel in +// cache.go and deleteEntryIf below; resurrecting a tombstone in place +// would invalidate deleteEntryIf's locked predicate). The overflow chain +// handles hash-equal keys (full hash collisions). type trieEntry[K comparable, V any] struct { trieNode[K, V] overflow atomic.Pointer[trieEntry[K, V]] From b200842459969b09e71a222788603bd094f7b1ac Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 00:11:15 -0600 Subject: [PATCH 11/21] cache: dedicated Clean claim sentinel; reclaim born-expired entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third audit round, independently verified (interleaving traces of every expires-word consumer against the new sentinel; mutation check: the regression tests fail against the reverted implementation): - Clean now claims an entry's expiry word with a dedicated sentinel (expiredClaimed, -2) instead of -1. -1 is the born-expired sentinel, which newStale deliberately re-anchors at now — so a Get snapshotting a stale in the window between Clean's claim and its tombstone CAS would resurrect the certified-dead value as a fresh Stale for up to another MaxStaleAge. entMaybeNewStale treats a claimed word as "no stale". Pinned deterministically by performing Clean's claim by hand (TestClean_ClaimWindowDoesNotResurrectDeadStale) plus a probabilistic hammer against the real Clean. - Born-expired entries (MaxAge(0) / MaxErrorAge(0) store -1) are now reclaimable as soon as their stale dies: their grace window was computed as satAdd(-1, maxStaleAge), anchored at the process epoch, so stale-less dead entries (error churn under MaxErrorAge(0)) could not be reclaimed until process uptime exceeded MaxStaleAge — and never, for huge stale ages. Reclaim is gated on TryGet visibility: value expired and stale dead. - Idle extension re-samples the clock immediately before publishing, shrinking the revive-after-expiry window from "whole deschedule since the expiry check" to a few instructions; the residual window is documented as best effort on MaxIdleAge. - Doc accuracy: Swap/Set "load is canceled" -> "load's result is discarded (the miss function is not interrupted)"; TryGet docs now cover expired-with-valid-stale returning Stale; MaxStaleAge accepts any negative as "forever" and documents the store-anchored stale window of Expired-but-unrefreshed keys; Item/Set Clean docs match Cache.Clean's actual rules. --- README.md | 2 +- cache/cache.go | 200 ++++++++++++++++++++++++++------------ cache/cache_test.go | 73 ++++++++++++++ cache/concurrency_test.go | 106 ++++++++++++++++++++ 4 files changed, 320 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 099c82a..57d0858 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ fails. Keys can be manually expired with `Expire`. Internally expired values or errors can be occasionally cleaned with `Clean`. Out of an abundance of paranoia that this code is correct, the test suite -layers several kinds of verification: unit tests at ~98% statement coverage +layers several kinds of verification: unit tests at ~99% statement coverage of the cache and trie (the remainder being defensive panics at invariant-violation points and race-retry branches that require interleavings that cannot be forced from userspace); all of the stdlib diff --git a/cache/cache.go b/cache/cache.go index d55ce27..9e5ea43 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -103,6 +103,18 @@ const ( stateFinalized ) +// Sentinels for loading.expires beyond 0 (in flight, or once finalized, +// "never expires"). Positive values are monotonic-clock nanos (see epoch); +// negative values are expired at every clock sample, since now is always +// strictly positive. The two negatives are deliberately distinct: newStale +// re-anchors a born-expired value's stale at now, which must not happen to +// a value whose stale Clean has already certified dead (see +// entMaybeNewStale). +const ( + expiredBorn = -1 // newExpires: MaxAge(0) / MaxErrorAge(0), expired at birth + expiredClaimed = -2 // Clean's claim: value expired and stale certified dead +) + // ent is the concrete type of cache entries living in the trie. Using a // type alias (Go 1.24+) lets us refer to entries by a short name while the // trie stays a generic hash-trie with no cache-specific logic. @@ -122,9 +134,10 @@ var epoch = time.Now().Add(-time.Second) func now() int64 { return int64(time.Since(epoch)) } // satAdd returns a+b saturated at math.MaxInt64, for expiry arithmetic on -// nanosecond quantities; b must be non-negative (a may be the -1 expired -// sentinel). Without saturation, a huge MaxAge / MaxStaleAge / MaxIdleAge -// would wrap negative and make entries born (or instantly) expired. +// nanosecond quantities; b must be non-negative, and callers pass a > 0 +// (sentinel expiries are filtered before the math). Without saturation, a +// huge MaxAge / MaxStaleAge / MaxIdleAge would wrap negative and make +// entries born (or instantly) expired. func satAdd(a, b int64) int64 { if s := a + b; s >= a { return s @@ -146,7 +159,7 @@ func (cfg *cfg) newExpires(err error) int64 { del0 = cfg.ageSet && cfg.maxAge <= 0 } if del0 { - return -1 + return expiredBorn } if ttl == 0 { return 0 @@ -174,9 +187,14 @@ func MaxAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxAge, c.a // value is the previous successfully cached value that is returned while the // value is being refreshed (a new value is being queried). As well, the stale // value is returned while the refreshed value is erroring. Without MaxAge, -// stales still apply to manually Expired keys and to erroring refreshes. +// stales still apply to manually Expired keys and to erroring refreshes; +// note that until a Get drives a refresh, an Expired key's stale window is +// anchored at the time the value was stored (a TryGet more than MaxStaleAge +// after the store finds no stale), while the refresh a Get drives re-anchors +// the window at the Expire. // -// A special value of -1 allows stale values to be returned indefinitely. +// A negative value (canonically -1) allows stale values to be returned +// indefinitely. func MaxStaleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxStaleAge = age }} } // MaxErrorAge sets the age to persist load errors. If not specified, the @@ -198,7 +216,12 @@ func MaxErrorAge(age time.Duration) Opt { // // The reset is best effort under races: a concurrent Expire, or a Clean // that has already deemed the entry expired, wins over an in-flight -// extension. +// extension. In the other direction, an extension computed just before the +// entry's expiry passed can land just after it and briefly revive the entry +// for another idle window. The clock is re-sampled immediately before an +// extension is published, so the revival window is a few instructions wide +// (stretched only by the extending goroutine being descheduled exactly +// there), but it is not fully closed. func MaxIdleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxIdleAge = age }} } @@ -359,9 +382,10 @@ outer: } // TryGet returns the value for the given key if it is cached. This returns -// either the currently stored value, or the current stale if the load errored, -// or the load error if there is no stale. If nothing is cached, or what is -// cached is expired, this returns Miss. +// either the currently stored value, or the current stale if the latest load +// errored or the value has expired while its stale is still valid, or the +// load error if there is no stale. If nothing is cached, or what is cached is +// expired with no valid stale, this returns Miss. func (c *Cache[K, V]) TryGet(k K) (v V, err error, _ KeyState) { e := c.t.loadEntry(k) if e == nil { @@ -436,10 +460,11 @@ func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // Clean deletes all expired values from the cache. A value is expired if // MaxAge is used and the entry is older than the max age, or if you manually -// expired a key. If MaxStaleAge is used and not -1, the entry must be older -// than MaxAge + MaxStaleAge, and an entry whose stale value is still within -// its window is never removed. If MaxStaleAge is -1, Clean returns -// immediately. +// expired a key. If MaxStaleAge is used and not negative, the entry must be +// older than MaxAge + MaxStaleAge, and an entry whose stale value is still +// within its window is never removed. Entries that were born expired +// (MaxAge(0) or MaxErrorAge(0)) are removed as soon as they have no valid +// stale. If MaxStaleAge is negative, Clean returns immediately. // // Clean also physically removes entries that were tombstoned by prior Delete // calls, so the trie's memory footprint does not grow unboundedly with @@ -463,22 +488,36 @@ func (c *Cache[K, V]) Clean() { return true } expires := l.expires.Load() - // The stale check matters for born-expired entries (MaxAge(0) / - // MaxErrorAge(0) store the -1 sentinel): their grace window would - // otherwise anchor at the epoch rather than at the entry, evicting - // stales that TryGet would still serve. For normally expired - // entries the stale dies at expires+maxStaleAge anyway, so the - // check changes nothing. - if expires != 0 && tn > satAdd(expires, int64(c.cfg.maxStaleAge)) && - (l.stale == nil || l.stale.expired(tn)) { + // An entry is reclaimable only once it is invisible to every + // read: the value must be expired and any stale must be out of + // its window (TryGet would return Miss). The stale check is also + // what anchors born-expired entries (MaxAge(0) / MaxErrorAge(0) + // store the expiredBorn sentinel): they have no entry-anchored + // expiry to add the grace window to — satAdd(-1, maxStaleAge) + // would anchor the grace at the process epoch, blocking reclaim + // of dead stale-less entries (error churn under MaxErrorAge(0)) + // until the process itself is older than MaxStaleAge, and forever + // for huge stale ages. Their stale, the only birth-anchored + // datum, gates them instead: reclaim exactly when TryGet + // visibility ends. Entries with a positive expiry keep the usual + // entry-anchored MaxAge + MaxStaleAge grace. + if expires != 0 && (l.stale == nil || l.stale.expired(tn)) && + (expires < 0 || tn > satAdd(expires, int64(c.cfg.maxStaleAge))) { // Win the expiry word before tombstoning: idle extension // CASes against the expiry it observed, so claiming the - // word here (writing the expired sentinel) makes a racing - // extension fail — the same way it loses to Expire — rather - // than be silently thrown away by our eviction. If instead - // the extension wins, the entry is in use; leave it for a - // later pass. - if l.expires.CompareAndSwap(expires, -1) && e.p.CompareAndSwap(l, nil) { + // word here makes a racing extension fail — the same way it + // loses to Expire — rather than be silently thrown away by + // our eviction. If instead the extension wins, the entry is + // in use; leave it for a later pass. + // + // The claim must be expiredClaimed, not expiredBorn: a + // racing Get that snapshots this loading for a stale in the + // window between our two CASes would read a -1 word as + // "born expired" and re-anchor the (certified dead) stale + // at now, serving the dead value as a fresh Stale for up to + // another MaxStaleAge. entMaybeNewStale treats a claimed + // word as "no stale". + if l.expires.CompareAndSwap(expires, expiredClaimed) && e.p.CompareAndSwap(l, nil) { toPrune = append(toPrune, e.key) } } @@ -511,10 +550,10 @@ func (c *Cache[K, V]) StopAutoClean() { } // Swap sets a value for a key. If the key is currently loading via Get, the -// load is canceled and Get returns the value from Swap. This returns the -// previously stored value, or the previous stale if the load errored, or the -// previous error if there is no stale. If nothing was cached, this returns -// Miss. +// load's result is discarded (the miss function itself is not interrupted) +// and Get returns the value from Swap. This returns the previously stored +// value, or the previous stale if the load errored, or the previous error if +// there is no stale. If nothing was cached, this returns Miss. func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { l := c.finalizedLoading(v) @@ -616,7 +655,8 @@ func (l *loading[V]) setve(v V, err error, expires int64) { } // Set sets a value for a key. If the key is currently loading via Get, the -// load is canceled and Get returns the value from Set. +// load's result is discarded (the miss function itself is not interrupted) +// and Get returns the value from Set. func (c *Cache[K, V]) Set(k K, v V) { c.Swap(k, v) } @@ -697,6 +737,11 @@ func (c *Cache[K, V]) finalizedLoading(v V) *loading[V] { // newStale handles expires <= 0 by basing the stale lifespan // on now, so an infinite-TTL + MaxStaleAge combo produces a // usable (non-epoch-born) stale for manual Expire to surface. + // The window is anchored here, at creation: a TryGet more + // than MaxStaleAge after this store finds the stale already + // dead even if the Expire was recent. A Get-driven + // replacement re-snapshots against the Expire's timestamp + // (see entMaybeNewStale) and surfaces a fresh window. l.stale = newStale(v, expires, c.cfg.maxStaleAge) } } @@ -756,6 +801,7 @@ func (s *stale[V]) expired(n int64) bool { // - if no stale age, we are not using stales, return nil // - if the loading is still pending, return nil (no value to snapshot) // - if loading has an error, return the prior stale +// - if Clean claimed the loading (value and stale certified dead), return nil // - if age is < 0, return new unexpiring stale // - else, return new stale with prior expiry + stale age func entMaybeNewStale[K comparable, V any](e *ent[K, V], age time.Duration) *stale[V] { @@ -769,15 +815,26 @@ func entMaybeNewStale[K comparable, V any](e *ent[K, V], age time.Duration) *sta if l.err != nil { return l.stale } - return newStale(l.v, l.expires.Load(), age) + expires := l.expires.Load() + if expires == expiredClaimed { + // Clean claimed this loading: the value is past its full window + // and its stale was certified dead. Without this check the claim + // would read as "born expired" below, and newStale would re-anchor + // the dead value's stale at now — resurrecting it as a fresh Stale + // for up to another MaxStaleAge. + return nil + } + return newStale(l.v, expires, age) } // newStale returns a fresh stale; age must be non-zero. // // expires is the main entry's expiry nano. If it is <= 0 (either 0 meaning -// the main entry never expires, or -1 meaning it expired immediately), we -// base the stale's lifespan on now so the stale is not born expired at the -// Unix epoch. +// the main entry never expires, or expiredBorn meaning it expired +// immediately), we base the stale's lifespan on now so the stale is not born +// expired at the process epoch. Callers must filter expiredClaimed before +// calling: a claimed value's stale is certified dead and must not be +// re-anchored (see entMaybeNewStale). func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { if age < 0 { return &stale[V]{v: v} @@ -840,8 +897,16 @@ func entGet[K comparable, V any](e *ent[K, V], extend time.Duration) (v V, err e // changed, the other writer wins. Never extend an expired entry: // a waited-on load that finalized already expired (MaxAge(0) // collapsing, or a TTL shorter than the load itself) is returned - // as a courtesy, not resurrected. - l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) + // as a courtesy, not resurrected. The !expired gate evaluated at + // n, which may have gone stale if we were descheduled since: a + // CAS landing after the expiry passed would revive an entry that + // concurrent readers may have already reported as Miss. Re-sample + // the clock immediately before publishing; the instructions + // between this sample and the CAS remain best effort (see + // MaxIdleAge). + if now() < expires { + l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) + } } return l.v, l.err, Hit } @@ -896,9 +961,13 @@ func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, if extend > 0 && l.err == nil { // CAS so a concurrent Expire is not silently overwritten; see the // matching comment in entGet. This path is unreachable when - // expired (the branch above returns), so unlike entGet no - // explicit !expired gate is needed. - l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) + // expired at n (the branch above returns), so unlike entGet no + // explicit !expired gate is needed — but n may have gone stale if + // we were descheduled since it was sampled, so re-sample the + // clock immediately before publishing, as in entGet. + if now() < expires { + l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) + } } return l.v, l.err, Hit } @@ -932,10 +1001,11 @@ func (i *Item[V]) Get(miss func() (V, error)) (v V, err error, state KeyState) { return i.c.Get(struct{}{}, miss) } -// TryGet returns the value the item if it is cached. This returns either the -// currently loaded value, or the current stale if the load errored, or the -// load error if there is no stale. If nothing is cached, or what is cached is -// expired, this returns Miss. +// TryGet returns the item's value if it is cached. This returns either the +// currently loaded value, or the current stale if the latest load errored or +// the value has expired while its stale is still valid, or the load error if +// there is no stale. If nothing is cached, or what is cached is expired with +// no valid stale, this returns Miss. func (i *Item[V]) TryGet() (v V, err error, state KeyState) { return i.c.TryGet(struct{}{}) } @@ -958,13 +1028,15 @@ func (i *Item[V]) Expire() { } // Set sets the value for the item. If the item is currently loading via Get, -// the load is canceled and Get returns the value from Set. +// the load's result is discarded (the miss function itself is not +// interrupted) and Get returns the value from Set. func (i *Item[V]) Set(v V) { i.c.Set(struct{}{}, v) } -// Swap sets the value for the item. If the item is currently loading via Get, -// the load is canceled and Get returns the value from Set. This returns the +// Swap sets the value for the item. If the item is currently loading via +// Get, the load's result is discarded (the miss function itself is not +// interrupted) and Get returns the value from Swap. This returns the // previously stored value, or the previous stale if the load errored, or the // previous error if there is no stale. If nothing is cached, this returns // Miss. @@ -993,8 +1065,11 @@ func (i *Item[V]) Clear() { // Clean deletes the item if it is expired. The item is expired if MaxAge is // used and the item is older than the max age, or if you manually expired -// it. If MaxStaleAge is used and not -1, the item must be older than MaxAge -// + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. +// it. If MaxStaleAge is used and not negative, the item must be older than +// MaxAge + MaxStaleAge, and an item whose stale value is still within its +// window is never removed. An item that was born expired (MaxAge(0) or +// MaxErrorAge(0)) is removed as soon as it has no valid stale. If +// MaxStaleAge is negative, Clean returns immediately. func (i *Item[V]) Clean() { i.c.Clean() } @@ -1038,9 +1113,10 @@ func (s *Set[K]) Get(k K, miss func() error) (err error, state KeyState) { } // TryGet returns the cached state for the given key: nil if the key loaded -// successfully, a stale nil if the latest load errored but a valid stale -// exists, or the load error itself. If nothing is cached, or what is cached -// is expired, this returns Miss. +// successfully, a stale nil if a valid stale exists while the latest load +// errored or the key expired, or the load error itself. If nothing is +// cached, or what is cached is expired with no valid stale, this returns +// Miss. func (s *Set[K]) TryGet(k K) (err error, state KeyState) { _, err, state = s.c.TryGet(k) return err, state @@ -1071,16 +1147,20 @@ func (s *Set[K]) Range(fn func(K, error) bool) { }) } -// Clean deletes all expired keys from the cache. A key is expired if MaxAge is -// used and the key is older than the max age, or if you manually expired a -// key. If MaxStaleAge is used and not -1, the entry must be older than MaxAge -// + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. +// Clean deletes all expired keys from the cache. A key is expired if MaxAge +// is used and the key is older than the max age, or if you manually expired +// a key. If MaxStaleAge is used and not negative, the entry must be older +// than MaxAge + MaxStaleAge, and an entry whose stale value is still within +// its window is never removed. Entries that were born expired (MaxAge(0) or +// MaxErrorAge(0)) are removed as soon as they have no valid stale. If +// MaxStaleAge is negative, Clean returns immediately. func (s *Set[K]) Clean() { s.c.Clean() } -// Set sets a key to exist. If the key is currently loading via Get, -// the load is canceled and Get returns nil. +// Set sets a key to exist. If the key is currently loading via Get, the +// load's result is discarded (the miss function itself is not interrupted) +// and Get returns nil. func (s *Set[K]) Set(k K) { s.c.Set(k, struct{}{}) } diff --git a/cache/cache_test.go b/cache/cache_test.go index 5330ee5..791a9b3 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -519,6 +519,79 @@ func TestClean_RespectsValidStales(t *testing.T) { } } +// trieEntries counts every entry wired into the trie, including tombstoned +// shells and entries that are invisible to Range and TryGet. +func trieEntries[K comparable, V any](c *Cache[K, V]) (n int) { + c.t.walk(func(*ent[K, V]) bool { n++; return true }) + return n +} + +// TestClean_ReclaimsBornExpired verifies Clean's handling of born-expired +// entries (the expiredBorn sentinel: MaxAge(0) / MaxErrorAge(0)), whose -1 +// expiry word carries no entry-anchored timeline to add the MaxStaleAge +// grace window to. Reclaim is gated on the stale instead: such an entry is +// removed exactly when no valid stale remains (i.e. as soon as TryGet +// reports Miss), regardless of process uptime. +// +// Regression test: the grace window used to be computed as +// satAdd(-1, maxStaleAge), anchoring it at the process epoch — born-expired +// stale-less entries (error churn under MaxErrorAge(0)) were unreclaimable +// until process uptime exceeded MaxStaleAge, and forever for huge stale +// ages. +func TestClean_ReclaimsBornExpired(t *testing.T) { + t.Run("errored_no_stale", func(t *testing.T) { + c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Hour)) + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: errored entry should be invisible, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d dead invisible born-expired entries in the trie", n) + } + }) + + t.Run("huge_stale_age", func(t *testing.T) { + c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Duration(math.MaxInt64))) + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: errored entry should be invisible, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d dead entries (huge MaxStaleAge made them permanent)", n) + } + }) + + t.Run("keeps_valid_stale", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) // born expired; stale window anchored at the store + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + c.Clean() + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("Clean evicted a born-expired entry with a valid stale: TryGet=(%d,%v)", v, s) + } + if n := trieEntries(c); n != 1 { + t.Fatalf("trie entries = %d, want 1", n) + } + }) + + t.Run("reclaims_dead_stale", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(5*time.Millisecond)) + c.Set("k", 1) + time.Sleep(10 * time.Millisecond) // stale window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: want Miss, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d entries whose stale window had passed", n) + } + }) +} + // TestCompareAndSwapAgreesWithTryGet verifies that CompareAndSwap and // CompareAndDelete only match live values: expired entries and errored // entries are "not there" per TryGet, so comparing against them must fail. diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index 0558ab3..57c2e50 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -955,3 +955,109 @@ func TestExpire_NotLostToIdleExtension(t *testing.T) { } } } + +// TestClean_ClaimWindowDoesNotResurrectDeadStale pins Clean's claim protocol +// against racing Gets: Clean claims an entry's expiry word before +// tombstoning, and a Get that lands between the claim and the tombstone +// snapshots that loading for a stale. The claim must not read as "born +// expired": entMaybeNewStale would re-anchor the certified-dead stale at now +// and the Get would serve a value past MaxAge+MaxStaleAge as a fresh Stale +// for up to another MaxStaleAge. +// +// The test performs Clean's claim by hand (same gate, same CAS) so the +// window between Clean's two CASes is held open deterministically. +// +// Regression test: Clean used to claim with -1, the born-expired sentinel, +// which is exactly what newStale re-anchors. +func TestClean_ClaimWindowDoesNotResurrectDeadStale(t *testing.T) { + c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) + c.Set("k", 1) + time.Sleep(5 * time.Millisecond) // expired AND past the stale window + + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: entry should be fully dead, got %v", s) + } + + // Clean's claim, by hand: same gate, same CAS, protocol frozen between + // Clean's two CASes. + e := c.t.loadEntry("k") + l := e.p.Load() + expires := l.expires.Load() + tn := now() + if expires <= 0 || tn <= satAdd(expires, int64(c.cfg.maxStaleAge)) || + (l.stale != nil && !l.stale.expired(tn)) { + t.Fatalf("precondition: entry not Clean-eligible: expires=%d", expires) + } + if !l.expires.CompareAndSwap(expires, expiredClaimed) { + t.Fatal("claim CAS failed") + } + + // A Get in the claim window must not serve the dead value as a stale: + // the stale snapshot must come up empty, so the Get blocks on (and + // returns) the fresh load. + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get in Clean's claim window: v=%d s=%v, want 2 Miss (a Stale here resurrects a certified-dead value)", v, s) + } + if l2 := e.p.Load(); l2 == nil || l2.stale != nil { + t.Fatalf("the replacement loading must carry no stale snapshot of the claimed value, got %+v", l2) + } + + // Finish Clean's protocol: the tombstone CAS must fail (the Get + // replaced the slot) and the fresh value must survive. + if e.p.CompareAndSwap(l, nil) { + t.Fatal("Clean's tombstone CAS should have failed against the Get's replacement") + } + if v, _, s := c.TryGet("k"); v != 2 || !s.IsHit() { + t.Fatalf("after claim window: TryGet=(%d,%v), want (2,Hit)", v, s) + } +} + +// TestClean_ClaimWindowVsGetRace hammers the window between Clean's two +// CASes (the expiry-word claim and the tombstone) with racing Gets. Every +// round populates keys whose value AND stale are certifiably dead (TryGet +// returned Miss), then races a real Clean against a Get scan: any Get that +// returns Stale resurrected a certified-dead value. +// +// Probabilistic tripwire for the protocol pinned deterministically by +// TestClean_ClaimWindowDoesNotResurrectDeadStale; before Clean claimed with +// a dedicated sentinel, this fired within a few seconds under -race. +func TestClean_ClaimWindowVsGetRace(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + const ( + keys = 1024 + // Small enough that entries die within a round's sleep; large + // enough that a wrongly re-anchored stale stays observable for the + // Get that snapshotted it. + staleAge = 5 * time.Millisecond + ) + c := New[int, int](MaxAge(time.Nanosecond), MaxStaleAge(staleAge)) + miss := func() (int, error) { return 2, nil } + for start := time.Now(); time.Since(start) < 3*time.Second; { + for k := range keys { + c.Set(k, 1) + } + time.Sleep(staleAge + 2*time.Millisecond) + for k := range keys { + for { + if _, _, s := c.TryGet(k); s.IsMiss() { + break // certified: expired and stale dead + } + } + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + c.Clean() + }() + for k := range keys { + if v, _, s := c.Get(k, miss); s == Stale { + t.Fatalf("key %d: Get racing Clean returned a certified-dead value as a stale: v=%d", k, v) + } + } + wg.Wait() + } +} From be30581ed98a987427f88f47320e8305aa39e4a5 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 12:59:29 -0600 Subject: [PATCH 12/21] cache: derive stale windows at read time; Expire never moves expiry forward MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth audit round (expiry/stale state machine), independently verified: all seven of the audit's repro tests were confirmed red against b200842 and are green now; eleven regression tests are ported into the suite. The round found three root causes composing into five bugs, two High: - The stale field conflated two meanings: the refresh companion (the previous generation, served while a replacement loads or errors) and a value's own post-expiry grace window. Only Set/Swap values got a self window (finalizedLoading aliased the companion to the value); Get- loaded values had none — TryGet missed at the instant of expiry while Get served a stale for the same state — or carried the previous generation, letting TryGet serve an older value than the freshest dead one. Idle extensions moved the expiry but not the frozen window, so long-lived entries died with their promised window already closed. The self window is now derived at read time from the expires word (staleOpen: anchor + MaxStaleAge, where anchor is the expiry, the birth for born-expired values, or the Expire timestamp), used identically by entGet, loadingTryGet, Swap's classification, and Clean's reclaim gate, so every read and refresh agrees by construction. The stale field reverts to a pure refresh companion (in-flight and errored reads only); finalizedLoading stores none. - Born-expired words (-1) destroyed the only datum that could anchor their window: entMaybeNewStale re-anchored a certified-dead value's stale at every Get-driven refresh, serving values of unbounded age as Stale (TryGet: Miss, Get: Stale, no race required). The word now carries the negated birth (-now(); the epoch pad keeps it clear of expiredClaimed), and the window anchors at the birth everywhere. - Expire blindly stored now()-1, moving already-expired words forward — re-anchoring dead stale windows (the invalidation API extending data lifetime) and overwriting Clean's claim sentinel inside its claim window. Expire now CASes and only ever moves the word backward; expiring an expired entry is a no-op, documented. Documented, not changed: Delete/Swap may report a concurrently finalizing load as the removed value (closing it costs more than the return triple is worth), and under MaxErrorAge(0) collapsed waiters on an erroring load re-drive serially (the literal "errors are never cached" contract; sharing completed errors is a design change deserving its own decision). Plus doc accuracy: MaxStaleAge's uniform anchoring, MaxIdleAge shortening on access, errors-cached-forever default, and the expires-word grammar at expiredClaimed. --- cache/cache.go | 402 ++++++++++++++++++++++++-------------- cache/cache_test.go | 188 ++++++++++++++++-- cache/concurrency_test.go | 168 +++++++++++++++- 3 files changed, 595 insertions(+), 163 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 9e5ea43..1262ad3 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -51,11 +51,21 @@ type ( expires int64 // nano at which this stale ent is unusable, if non-zero } loading[V any] struct { - v V - err error - expires atomic.Int64 // nano at which this ent is unusable, if non-zero - stale *stale[V] - state atomic.Uint32 + v V + err error + // expires is the entry's expiry word; see the grammar documented + // at expiredClaimed below. + expires atomic.Int64 + // stale is the refresh companion: the previous generation's value, + // snapshotted when this loading was installed to replace an + // expired or errored predecessor (entMaybeNewStale), and served + // while this loading is in flight or while its result is an + // error. It is immutable once the loading is published. An + // expired err-free value's own post-expiry stale window is not + // stored anywhere; it is derived from the expires word at read + // time (see staleOpen). + stale *stale[V] + state atomic.Uint32 wg sync.WaitGroup mu sync.Mutex @@ -103,17 +113,25 @@ const ( stateFinalized ) -// Sentinels for loading.expires beyond 0 (in flight, or once finalized, -// "never expires"). Positive values are monotonic-clock nanos (see epoch); -// negative values are expired at every clock sample, since now is always -// strictly positive. The two negatives are deliberately distinct: newStale -// re-anchors a born-expired value's stale at now, which must not happen to -// a value whose stale Clean has already certified dead (see +// The grammar of the loading.expires word. Positive values are +// monotonic-clock nanos (see epoch); all negative values are expired at +// every clock sample, since now is always strictly positive: +// +// 0: load in flight, or — once finalized — "never expires" +// > 0: expires at that nano +// -2: expiredClaimed, Clean's claim: the value is expired and its +// stale window is certified closed +// < -2: born expired (MaxAge(0) / MaxErrorAge(0)), carrying the negated +// birth nano so the value's stale window can anchor at the birth +// (see newExpires, staleOpen, newStale) +// +// The epoch's one-second pad keeps now() at or above a second of nanos, so +// a negated birth can never collide with expiredClaimed. The two negative +// shapes are deliberately distinct: a born-expired value's stale window is +// derived from its birth, while a claimed word means the window was +// certified closed and must never be re-derived (see staleOpen and // entMaybeNewStale). -const ( - expiredBorn = -1 // newExpires: MaxAge(0) / MaxErrorAge(0), expired at birth - expiredClaimed = -2 // Clean's claim: value expired and stale certified dead -) +const expiredClaimed = -2 // ent is the concrete type of cache entries living in the trie. Using a // type alias (Go 1.24+) lets us refer to entries by a short name while the @@ -126,7 +144,8 @@ type ent[K comparable, V any] = trieEntry[K, loading[V]] // races against idle extension — relies on that, and a wall-clock step // (NTP, manual adjustment) would otherwise move every TTL. The one-second // pad keeps now strictly positive so it can never collide with the 0 -// sentinels ("no expiry" / "caller samples the clock"). Trade-off: on +// sentinels ("no expiry" / "caller samples the clock"), and keeps negated +// births (see the expires-word grammar) below expiredClaimed. Trade-off: on // platforms whose monotonic clock pauses during system suspend, entries // age only while the system runs. var epoch = time.Now().Add(-time.Second) @@ -159,7 +178,12 @@ func (cfg *cfg) newExpires(err error) int64 { del0 = cfg.ageSet && cfg.maxAge <= 0 } if del0 { - return expiredBorn + // Born expired: negative, so the value is expired at every clock + // sample, carrying the negated birth so its stale window anchors + // at the birth rather than at whatever read happens to derive it + // (see staleOpen, newStale). The epoch pad keeps this below + // expiredClaimed. + return -now() } if ttl == 0 { return 0 @@ -187,24 +211,34 @@ func MaxAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxAge, c.a // value is the previous successfully cached value that is returned while the // value is being refreshed (a new value is being queried). As well, the stale // value is returned while the refreshed value is erroring. Without MaxAge, -// stales still apply to manually Expired keys and to erroring refreshes; -// note that until a Get drives a refresh, an Expired key's stale window is -// anchored at the time the value was stored (a TryGet more than MaxStaleAge -// after the store finds no stale), while the refresh a Get drives re-anchors -// the window at the Expire. +// stales still apply to manually Expired keys and to erroring refreshes. +// +// The stale window is anchored at the moment the value expired — its TTL +// (or idle extension) lapsing, its birth for values born expired +// (MaxAge(0)), or the Expire call that killed it — and every read agrees +// on it: TryGet serves the stale for exactly as long as a Get-driven +// refresh would, and a value whose window has closed is never served +// again. // // A negative value (canonically -1) allows stale values to be returned // indefinitely. func MaxStaleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxStaleAge = age }} } // MaxErrorAge sets the age to persist load errors. If not specified, the -// default is MaxAge. Using this option with 0 disables caching errors -// entirely. +// default is MaxAge — so with neither option set, a load error is cached +// forever and never retried (until Set, Swap, Delete, or Expire). Using +// this option with 0 disables caching errors entirely. // // A cached error suppresses repeat queries only when there is no stale // value to return: if stale values are enabled and a valid stale exists, // Get returns the stale and always drives a refresh, regardless of the // error's remaining age. +// +// With errors uncached (age 0), concurrent Gets that collapsed onto a load +// that then errors do not share the error: only the Get that drove the +// load returns it, and each collapsed waiter re-drives its own load in +// turn, so one erroring wave of N collapsed Gets can issue up to N +// sequential loads. func MaxErrorAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxErrAge, c.errAgeSet = age, true }} } @@ -212,7 +246,9 @@ func MaxErrorAge(age time.Duration) Opt { // MaxIdleAge opts in to extending an entry's expiry on each successful // access. Each time Get or TryGet returns a live Hit without an error, the // entry's expiry is reset to now + age. If MaxAge is not set, the idle -// age is also used as the initial TTL. +// age is also used as the initial TTL. The reset applies in both +// directions: with MaxIdleAge smaller than MaxAge, an access shortens the +// entry's remaining life from the MaxAge-given expiry to now + age. // // The reset is best effort under races: a concurrent Expire, or a Clean // that has already deemed the entry expired, wins over an in-flight @@ -289,7 +325,7 @@ func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeySta // it without any locking. e := c.t.loadEntry(k) if e != nil { - if v, err, s = entGet(e, c.cfg.maxIdleAge); s == Hit { + if v, err, s = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); s == Hit { return v, err, s } } @@ -325,7 +361,7 @@ outer: // to wait for it (finalized hit), return a stale (load in // flight with a valid stale), or replace it (finalized but // expired or errored). - if v, err, s = entGet(e, c.cfg.maxIdleAge); s == Hit { + if v, err, s = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); s == Hit { return v, err, s } if s == Stale && !prev.finalized() && e.p.Load() == prev { @@ -371,8 +407,13 @@ outer: // been replaced by another writer. extend=0: this Get drove the load // and publicly returns Miss, so it is not an idle-extending access // (MaxIdleAge documents extension on Hits); the value keeps the - // initial TTL it was born with. - v, err, s = entGet(e, 0) + // initial TTL it was born with. staleAge=0: this Get returns the + // load's result, never a re-derivation of that result as a Stale (a + // short-TTL value can expire the instant it loads; its derived window + // serves later readers, not the load that produced it) — the only + // stale surfaced here is a refresh companion carried by a loading + // still in flight. + v, err, s = entGet(e, 0, 0) switch s { case Miss, Hit: l.wg.Wait() @@ -391,7 +432,7 @@ func (c *Cache[K, V]) TryGet(k K) (v V, err error, _ KeyState) { if e == nil { return v, err, Miss } - return entTryGet(e, 0, c.cfg.maxIdleAge) + return entTryGet(e, 0, c.cfg.maxIdleAge, c.cfg.maxStaleAge) } // Delete deletes the value for a key and returns the prior value, if stored @@ -400,7 +441,9 @@ func (c *Cache[K, V]) TryGet(k K) (v V, err error, _ KeyState) { // Delete physically removes the entry from the trie. If a concurrent Get is // holding a reference to the deleted entry via its in-flight loading, that // Get still completes with the miss function's result; a new Get arriving -// after Delete creates a fresh entry and may drive an independent miss. +// after Delete creates a fresh entry and may drive an independent miss. A +// load that finalizes concurrently with the removal may be reported as the +// removed value even though no read ever observed it cached. func (c *Cache[K, V]) Delete(k K) (v V, err error, _ KeyState) { e := c.t.loadEntry(k) if e == nil { @@ -424,7 +467,7 @@ func (c *Cache[K, V]) Delete(k K) (v V, err error, _ KeyState) { if was == nil { return v, err, Miss } - return loadingTryGet(was, n, 0) + return loadingTryGet(was, n, 0, c.cfg.maxStaleAge) } // Expire sets a stored value to expire immediately, meaning the next Get will @@ -433,14 +476,35 @@ func (c *Cache[K, V]) Delete(k K) (v V, err error, _ KeyState) { // // Expire only affects entries that have finished loading. Calling Expire for a // key whose miss function is still running is a no-op; the in-flight load will -// complete with its normal TTL and is not canceled or shortened. +// complete with its normal TTL and is not canceled or shortened. Expiring an +// already-expired entry is also a no-op: the expiry is never moved forward, +// so a dead value's stale window cannot be re-anchored or revived, and a +// concurrent Clean's claim is never overwritten. func (c *Cache[K, V]) Expire(k K) { e := c.t.loadEntry(k) if e == nil { return } - if l := entLoad(e); l != nil && l.finalized() { - l.expires.Store(now() - 1) + l := entLoad(e) + if l == nil || !l.finalized() { + return + } + // CAS rather than a blind store, and only ever backward in time. A + // blind store would move an already-expired word forward — re-anchoring + // the dead value's stale window at the Expire (resurrecting data the + // caller explicitly invalidated) and clobbering Clean's claim sentinel + // inside its claim window. The loop still beats a racing idle + // extension: if the extension's CAS lands first, ours fails, reloads + // the extended (live) expiry, and expires it. + for { + cur := l.expires.Load() + n := now() + if cur != 0 && cur <= n { + return // already expired (including born-expired and claimed) + } + if l.expires.CompareAndSwap(cur, n-1) { + return + } } } @@ -450,7 +514,7 @@ func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // current time when we enter range and avoid it in all tryGet calls. tn := now() c.t.walk(func(e *ent[K, V]) bool { - v, err, s := entTryGet(e, tn, 0) + v, err, s := entTryGet(e, tn, 0, c.cfg.maxStaleAge) if s.IsMiss() { return true } @@ -460,11 +524,13 @@ func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // Clean deletes all expired values from the cache. A value is expired if // MaxAge is used and the entry is older than the max age, or if you manually -// expired a key. If MaxStaleAge is used and not negative, the entry must be -// older than MaxAge + MaxStaleAge, and an entry whose stale value is still -// within its window is never removed. Entries that were born expired -// (MaxAge(0) or MaxErrorAge(0)) are removed as soon as they have no valid -// stale. If MaxStaleAge is negative, Clean returns immediately. +// expired a key. An entry is removed only once it is invisible to every +// read: with MaxStaleAge used and not negative, a value's stale window runs +// until MaxAge + MaxStaleAge — anchored at the birth for entries born +// expired (MaxAge(0) or MaxErrorAge(0)) — and an entry whose window is +// still open is never removed. Expired errors are removed as soon as no +// valid stale remains. If MaxStaleAge is negative, Clean returns +// immediately. // // Clean also physically removes entries that were tombstoned by prior Delete // calls, so the trie's memory footprint does not grow unboundedly with @@ -479,7 +545,7 @@ func (c *Cache[K, V]) Clean() { // physical removal (both newly-tombstoned and pre-existing tombstones). var toPrune []K c.t.walk(func(e *ent[K, V]) bool { - l := e.p.Load() + l := entLoad(e) if l == nil { toPrune = append(toPrune, e.key) return true @@ -489,20 +555,19 @@ func (c *Cache[K, V]) Clean() { } expires := l.expires.Load() // An entry is reclaimable only once it is invisible to every - // read: the value must be expired and any stale must be out of - // its window (TryGet would return Miss). The stale check is also - // what anchors born-expired entries (MaxAge(0) / MaxErrorAge(0) - // store the expiredBorn sentinel): they have no entry-anchored - // expiry to add the grace window to — satAdd(-1, maxStaleAge) - // would anchor the grace at the process epoch, blocking reclaim - // of dead stale-less entries (error churn under MaxErrorAge(0)) - // until the process itself is older than MaxStaleAge, and forever - // for huge stale ages. Their stale, the only birth-anchored - // datum, gates them instead: reclaim exactly when TryGet - // visibility ends. Entries with a positive expiry keep the usual - // entry-anchored MaxAge + MaxStaleAge grace. - if expires != 0 && (l.stale == nil || l.stale.expired(tn)) && - (expires < 0 || tn > satAdd(expires, int64(c.cfg.maxStaleAge))) { + // read, i.e. exactly when TryGet would return Miss: the value + // must be expired, any carried refresh-companion stale (servable + // while the result is an error) must be out of its window, and an + // err-free value's derived self-stale window must be closed. + // staleOpen mirrors the read paths, anchoring born-expired + // entries at their birth — so dead stale-less entries (error + // churn under MaxErrorAge(0)) are reclaimed immediately, and a + // huge MaxStaleAge cannot block their reclaim forever. Errored + // values have no self-stale: they are reclaimable as soon as + // they are expired with no valid companion. + expired := expires != 0 && expires <= tn + if expired && (l.stale == nil || l.stale.expired(tn)) && + (l.err != nil || !staleOpen(expires, tn, c.cfg.maxStaleAge)) { // Win the expiry word before tombstoning: idle extension // CASes against the expiry it observed, so claiming the // word here makes a racing extension fail — the same way it @@ -510,13 +575,10 @@ func (c *Cache[K, V]) Clean() { // our eviction. If instead the extension wins, the entry is // in use; leave it for a later pass. // - // The claim must be expiredClaimed, not expiredBorn: a - // racing Get that snapshots this loading for a stale in the - // window between our two CASes would read a -1 word as - // "born expired" and re-anchor the (certified dead) stale - // at now, serving the dead value as a fresh Stale for up to - // another MaxStaleAge. entMaybeNewStale treats a claimed - // word as "no stale". + // The claimed word also reads as "stale window certified + // closed" everywhere (staleOpen, entMaybeNewStale), so a + // racing Get that snapshots this loading between our two + // CASes can never re-derive a window for the dead value. if l.expires.CompareAndSwap(expires, expiredClaimed) && e.p.CompareAndSwap(l, nil) { toPrune = append(toPrune, e.key) } @@ -554,6 +616,9 @@ func (c *Cache[K, V]) StopAutoClean() { // and Get returns the value from Swap. This returns the previously stored // value, or the previous stale if the load errored, or the previous error if // there is no stale. If nothing was cached, this returns Miss. +// +// A load that finalizes concurrently with the swap may be reported as the +// prior value even though no read ever observed it cached. func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { l := c.finalizedLoading(v) @@ -586,13 +651,26 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { } was.mu.Unlock() } - if expired := was.expired(n); was.err != nil || expired { + // Mirror loadingTryGet's classification so the displaced value is + // reported exactly as a TryGet at wasN would have reported it: a + // valid companion stale outranks an error, an unexpired error is + // itself the cached result, and an expired err-free value is + // still the prior value (Stale) while its derived window is open. + expires := was.expires.Load() + expired := expires != 0 && expires <= n + if was.err != nil { if was.stale != nil && !was.stale.expired(n) { old, oldState = was.stale.v, Stale + } else if !expired { + old, oldErr, oldState = was.v, was.err, Hit } - if expired { - return + return + } + if expired { + if staleOpen(expires, n, c.cfg.maxStaleAge) { + old, oldState = was.v, Stale } + return } old, oldErr, oldState = was.v, was.err, Hit }() @@ -727,23 +805,16 @@ func casMatches[V any](l *loading[V], old V) bool { /////////////////// func (c *Cache[K, V]) finalizedLoading(v V) *loading[V] { + // No stale companion: the companion is only ever read while a loading + // is in flight or errored, and a Set/Swap-created loading is born + // finalized with no error. The value's own post-expiry stale window is + // derived from the expires word at read time (see staleOpen), so it + // needs no snapshot here either. l := &loading[V]{ v: v, } - expires := c.cfg.newExpires(nil) - if expires != 0 || c.cfg.maxStaleAge != 0 { + if expires := c.cfg.newExpires(nil); expires != 0 { l.expires.Store(expires) - if c.cfg.maxStaleAge != 0 { - // newStale handles expires <= 0 by basing the stale lifespan - // on now, so an infinite-TTL + MaxStaleAge combo produces a - // usable (non-epoch-born) stale for manual Expire to surface. - // The window is anchored here, at creation: a TryGet more - // than MaxStaleAge after this store finds the stale already - // dead even if the Expire was recent. A Get-driven - // replacement re-snapshots against the Expire's timestamp - // (see entMaybeNewStale) and surfaces a fresh window. - l.stale = newStale(v, expires, c.cfg.maxStaleAge) - } } l.state.Store(stateFinalized) return l @@ -794,16 +865,18 @@ func (s *stale[V]) expired(n int64) bool { return expires != 0 && expires <= n } -// entMaybeNewStale derives a new stale snapshot from the entry's current -// loading, for use when installing a replacement loading. +// entMaybeNewStale derives a stale snapshot from the entry's current +// loading, for use as the refresh companion of a replacement loading. // // - if entry has no live loading, no stale, return nil // - if no stale age, we are not using stales, return nil // - if the loading is still pending, return nil (no value to snapshot) -// - if loading has an error, return the prior stale -// - if Clean claimed the loading (value and stale certified dead), return nil +// - if loading has an error, return the prior stale (companion passthrough) +// - if Clean claimed the loading (window certified closed), return nil // - if age is < 0, return new unexpiring stale -// - else, return new stale with prior expiry + stale age +// - else, return a stale spanning the value's derived window: its expiry +// (or birth, for born-expired values) plus the stale age — the same +// window staleOpen computes on the read paths func entMaybeNewStale[K comparable, V any](e *ent[K, V], age time.Duration) *stale[V] { if age == 0 { return nil @@ -817,30 +890,55 @@ func entMaybeNewStale[K comparable, V any](e *ent[K, V], age time.Duration) *sta } expires := l.expires.Load() if expires == expiredClaimed { - // Clean claimed this loading: the value is past its full window - // and its stale was certified dead. Without this check the claim - // would read as "born expired" below, and newStale would re-anchor - // the dead value's stale at now — resurrecting it as a fresh Stale - // for up to another MaxStaleAge. + // Clean claimed this loading: the value is past its full window, + // which Clean certified closed. Never re-derive a window for it. return nil } return newStale(l.v, expires, age) } +// staleOpen reports whether the derived self-stale window of an expired, +// err-free loading is still open at n: the same window newStale builds +// when the value is snapshotted as a refresh companion, so every read +// agrees with every refresh about how long an expired value remains +// servable. expires must be the loading's non-zero expiry word: positive +// is the expiry itself, a negated birth anchors born-expired values at +// their birth, and a Clean-claimed word means the window was certified +// closed. +func staleOpen(expires, n int64, age time.Duration) bool { + if age == 0 || expires == expiredClaimed { + return false + } + if age < 0 { + return true + } + anchor := expires + if anchor < 0 { + anchor = -anchor // born expired: the negated birth (see newExpires) + } + return n < satAdd(anchor, int64(age)) +} + // newStale returns a fresh stale; age must be non-zero. // -// expires is the main entry's expiry nano. If it is <= 0 (either 0 meaning -// the main entry never expires, or expiredBorn meaning it expired -// immediately), we base the stale's lifespan on now so the stale is not born -// expired at the process epoch. Callers must filter expiredClaimed before -// calling: a claimed value's stale is certified dead and must not be -// re-anchored (see entMaybeNewStale). +// expires is the main entry's expiry word. A negative word carries the +// value's negated birth (see newExpires) and anchors the window at the +// birth, matching staleOpen. A 0 word (the value never expires) anchors at +// now; it is reachable only when a racing writer replaced the slot between +// the caller's expiry check and our snapshot, and the caller's install CAS +// then fails and discards the snapshot — but handle it sanely regardless. +// Callers must filter expiredClaimed before calling: a claimed value's +// window is certified closed and must not be re-derived (see +// entMaybeNewStale, staleOpen). func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { if age < 0 { return &stale[V]{v: v} } - if expires <= 0 { + switch { + case expires == 0: expires = now() + case expires < 0: + expires = -expires } return &stale[V]{v, satAdd(expires, int64(age))} } @@ -850,7 +948,7 @@ func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { // does not force a miss: Get must hand its caller something even if the // value expired the instant it loaded (e.g. request collapsing with // MaxAge(0)). -func entGet[K comparable, V any](e *ent[K, V], extend time.Duration) (v V, err error, state KeyState) { +func entGet[K comparable, V any](e *ent[K, V], extend, staleAge time.Duration) (v V, err error, state KeyState) { l := entLoad(e) var waited bool if l == nil { @@ -864,46 +962,52 @@ func entGet[K comparable, V any](e *ent[K, V], extend time.Duration) (v V, err e waited = true } - // If we did not wait and our entry is expired (value or error), or if - // our entry is not expired but has errored, we potentially return the - // stale entry. - // - // If we waited, we could immediately be expired due to time sync, or - // if the user is configured to never cache and they're just using - // request collapsing: we still want to return the now expired value. - // - // The expiry must be loaded before the clock is sampled: Expire stores - // now()-1 from its own clock, which can be ahead of a clock sampled + // The expiry must be loaded before the clock is sampled: Expire CASes + // in now()-1 from its own clock, which can be ahead of a clock sampled // before our load; sampling after the load guarantees a concurrent // Expire is classified as expired here rather than extended over. expires := l.expires.Load() n := now() expired := expires != 0 && expires <= n - if (!waited && expired) || l.err != nil { + if l.err != nil { + // Errored: a valid companion stale (the previous generation) + // outranks the error; with no companion, the error itself is the + // cached result while it lives, and a dead error is a miss. if l.stale != nil && !l.stale.expired(n) { return l.stale.v, nil, Stale } - // The stale value is expired: if our entry is not expired, - // this must be an error we waited on. if !expired { return l.v, l.err, Hit } return v, err, state } - if extend > 0 && l.err == nil && !expired { + if expired { + // If we waited, the load could finalize already expired (MaxAge(0) + // collapsing, or a TTL shorter than the load itself): hand back + // the now-expired value as a courtesy rather than re-drive. + // Otherwise the expired value is served as Stale while its + // derived window is open (see staleOpen), and is a miss after. + if waited { + return l.v, nil, Hit + } + if staleOpen(expires, n, staleAge) { + return l.v, nil, Stale + } + return v, err, state + } + if extend > 0 { // Extend the idle expiry via CAS against the expiry we evaluated // above so that a concurrent Expire (or Swap finalization, or a // Clean eviction) is not silently overwritten; if expires - // changed, the other writer wins. Never extend an expired entry: - // a waited-on load that finalized already expired (MaxAge(0) - // collapsing, or a TTL shorter than the load itself) is returned - // as a courtesy, not resurrected. The !expired gate evaluated at - // n, which may have gone stale if we were descheduled since: a - // CAS landing after the expiry passed would revive an entry that - // concurrent readers may have already reported as Miss. Re-sample - // the clock immediately before publishing; the instructions - // between this sample and the CAS remain best effort (see - // MaxIdleAge). + // changed, the other writer wins. This path is structurally + // unexpired at n — a waited-on load that finalized already + // expired was returned above as a courtesy, not resurrected — + // but n may have gone stale if we were descheduled since it was + // sampled: a CAS landing after the expiry passed would revive an + // entry that concurrent readers may have already reported as + // Miss. Re-sample the clock immediately before publishing; the + // instructions between this sample and the CAS remain best + // effort (see MaxIdleAge). if now() < expires { l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) } @@ -911,18 +1015,18 @@ func entGet[K comparable, V any](e *ent[K, V], extend time.Duration) (v V, err e return l.v, l.err, Hit } -func entTryGet[K comparable, V any](e *ent[K, V], n64 int64, extend time.Duration) (v V, err error, state KeyState) { +func entTryGet[K comparable, V any](e *ent[K, V], n64 int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { l := entLoad(e) if l == nil { return v, err, state } - return loadingTryGet(l, n64, extend) + return loadingTryGet(l, n64, extend, staleAge) } // loadingTryGet is entTryGet for a loading already plucked from an entry; // Delete uses it directly on the loading it captured while tombstoning, so // the value it returns is exactly the value it removed. -func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, err error, state KeyState) { +func loadingTryGet[V any](l *loading[V], n64 int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { // Fast path: finalized, no expiry, no error — skip time checks. if l.finalized() && l.expires.Load() == 0 && l.err == nil { return l.v, nil, Hit @@ -939,7 +1043,6 @@ func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, return v, err, state } - // If we have an error or we are expired, we maybe return the stale. // The expiry must be loaded before the clock is sampled (when we are // the one sampling it); see the matching comment in entGet. A caller- // provided n64 (Range's batch timestamp, Delete's pre-removal clock) @@ -950,19 +1053,30 @@ func loadingTryGet[V any](l *loading[V], n64 int64, extend time.Duration) (v V, } n := n64 expired := expires != 0 && expires <= n - if l.err != nil || expired { + if l.err != nil { + // Errored: a valid companion stale (the previous generation) + // outranks the error; with no companion, the error itself is the + // cached result while it lives, and a dead error is a miss. if l.stale != nil && !l.stale.expired(n) { return l.stale.v, nil, Stale } if expired { return v, err, state } + return l.v, l.err, Hit + } + if expired { + // An expired value is served as Stale while its derived window + // is open (see staleOpen), and is a miss after. + if staleOpen(expires, n, staleAge) { + return l.v, nil, Stale + } + return v, err, state } - if extend > 0 && l.err == nil { + if extend > 0 { // CAS so a concurrent Expire is not silently overwritten; see the - // matching comment in entGet. This path is unreachable when - // expired at n (the branch above returns), so unlike entGet no - // explicit !expired gate is needed — but n may have gone stale if + // matching comment in entGet. This path is structurally unexpired + // at n (the branch above returns) — but n may have gone stale if // we were descheduled since it was sampled, so re-sample the // clock immediately before publishing, as in entGet. if now() < expires { @@ -1022,7 +1136,8 @@ func (i *Item[V]) Delete() (V, error, KeyState) { // // Expire only affects an item that has finished loading. Calling Expire while // the miss function is still running is a no-op; the in-flight load will -// complete with its normal TTL and is not canceled or shortened. +// complete with its normal TTL and is not canceled or shortened. Expiring an +// already-expired item is also a no-op: the expiry is never moved forward. func (i *Item[V]) Expire() { i.c.Expire(struct{}{}) } @@ -1065,11 +1180,12 @@ func (i *Item[V]) Clear() { // Clean deletes the item if it is expired. The item is expired if MaxAge is // used and the item is older than the max age, or if you manually expired -// it. If MaxStaleAge is used and not negative, the item must be older than -// MaxAge + MaxStaleAge, and an item whose stale value is still within its -// window is never removed. An item that was born expired (MaxAge(0) or -// MaxErrorAge(0)) is removed as soon as it has no valid stale. If -// MaxStaleAge is negative, Clean returns immediately. +// it. The item is removed only once it is invisible to every read: with +// MaxStaleAge used and not negative, its stale window runs until +// MaxAge + MaxStaleAge — anchored at the birth if the item was born expired +// (MaxAge(0) or MaxErrorAge(0)) — and it is never removed while the window +// is still open. An expired error is removed as soon as no valid stale +// remains. If MaxStaleAge is negative, Clean returns immediately. func (i *Item[V]) Clean() { i.c.Clean() } @@ -1135,7 +1251,8 @@ func (s *Set[K]) Delete(k K) (error, KeyState) { // // Expire only affects a key whose load has finished. Calling Expire while the // miss function is still running is a no-op; the in-flight load will complete -// with its normal TTL and is not canceled or shortened. +// with its normal TTL and is not canceled or shortened. Expiring an +// already-expired key is also a no-op: the expiry is never moved forward. func (s *Set[K]) Expire(k K) { s.c.Expire(k) } @@ -1149,11 +1266,12 @@ func (s *Set[K]) Range(fn func(K, error) bool) { // Clean deletes all expired keys from the cache. A key is expired if MaxAge // is used and the key is older than the max age, or if you manually expired -// a key. If MaxStaleAge is used and not negative, the entry must be older -// than MaxAge + MaxStaleAge, and an entry whose stale value is still within -// its window is never removed. Entries that were born expired (MaxAge(0) or -// MaxErrorAge(0)) are removed as soon as they have no valid stale. If -// MaxStaleAge is negative, Clean returns immediately. +// a key. An entry is removed only once it is invisible to every read: with +// MaxStaleAge used and not negative, its stale window runs until +// MaxAge + MaxStaleAge — anchored at the birth for entries born expired +// (MaxAge(0) or MaxErrorAge(0)) — and an entry whose window is still open +// is never removed. Expired errors are removed as soon as no valid stale +// remains. If MaxStaleAge is negative, Clean returns immediately. func (s *Set[K]) Clean() { s.c.Clean() } diff --git a/cache/cache_test.go b/cache/cache_test.go index 791a9b3..6832a0a 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -499,10 +499,10 @@ func TestHugeAges(t *testing.T) { } // TestClean_RespectsValidStales verifies Clean never removes an entry whose -// stale is still within its window. Born-expired entries (MaxAge(0) stores -// the -1 expired sentinel) are the regression case: their MaxAge+MaxStaleAge -// grace would otherwise anchor at the epoch instead of at the entry, so -// Clean evicted stales that TryGet was still serving. +// stale window is still open. Born-expired entries (MaxAge(0) stores a +// negative, birth-carrying word) are the regression case: their +// MaxAge+MaxStaleAge grace used to anchor at the epoch instead of at the +// entry, so Clean evicted stales that TryGet was still serving. func TestClean_RespectsValidStales(t *testing.T) { c := New[string, int](MaxAge(0), MaxStaleAge(50*time.Millisecond)) c.Set("k", 1) @@ -527,17 +527,17 @@ func trieEntries[K comparable, V any](c *Cache[K, V]) (n int) { } // TestClean_ReclaimsBornExpired verifies Clean's handling of born-expired -// entries (the expiredBorn sentinel: MaxAge(0) / MaxErrorAge(0)), whose -1 -// expiry word carries no entry-anchored timeline to add the MaxStaleAge -// grace window to. Reclaim is gated on the stale instead: such an entry is -// removed exactly when no valid stale remains (i.e. as soon as TryGet -// reports Miss), regardless of process uptime. +// entries (MaxAge(0) / MaxErrorAge(0), whose negative expiry word carries +// the negated birth): an entry is removed exactly when no read can see it +// anymore (i.e. as soon as TryGet reports Miss) — immediately for errored +// stale-less entries, and exactly when the birth-anchored stale window +// closes for values — regardless of process uptime. // -// Regression test: the grace window used to be computed as -// satAdd(-1, maxStaleAge), anchoring it at the process epoch — born-expired -// stale-less entries (error churn under MaxErrorAge(0)) were unreclaimable -// until process uptime exceeded MaxStaleAge, and forever for huge stale -// ages. +// Regression test: born-expired entries used to store a bare -1 sentinel, +// and the grace window was computed as satAdd(-1, maxStaleAge), anchoring +// it at the process epoch — born-expired stale-less entries (error churn +// under MaxErrorAge(0)) were unreclaimable until process uptime exceeded +// MaxStaleAge, and forever for huge stale ages. func TestClean_ReclaimsBornExpired(t *testing.T) { t.Run("errored_no_stale", func(t *testing.T) { c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Hour)) @@ -592,6 +592,166 @@ func TestClean_ReclaimsBornExpired(t *testing.T) { }) } +// TestStale_BornExpiredAnchorsAtBirth verifies that a born-expired value's +// (MaxAge(0)) stale window is anchored at the value's birth — carried by +// the negated-birth expiry word — for every read and refresh. Once the +// window passes, the value is gone for good: a later Get drives a fresh +// load rather than serving the dead value. +// +// Regression test: born-expired values used to store a bare -1 sentinel, +// destroying the birth; a Get-driven refresh arbitrarily later re-anchored +// the dead value's stale at the refresh (newStale's expires<=0 fallback), +// serving a value of unbounded age as a fresh Stale right after TryGet +// certified Miss. +func TestStale_BornExpiredAnchorsAtBirth(t *testing.T) { + const age = 50 * time.Millisecond + c := New[string, int](MaxAge(0), MaxStaleAge(age)) + c.Set("k", 1) + + // Within the window, the value is servable as a stale. + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet within the window: (%d, %v), want (1, Stale)", v, s) + } + + time.Sleep(2 * age) // window passed; the value is certifiably dead + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the window: %v, want Miss", s) + } + v, _, s := c.Get("k", func() (int, error) { return 2, nil }) + if v != 2 || s != Miss { + t.Fatalf("Get after the window: (%d, %v), want (2, Miss): the dead value must not be re-anchored at the refresh", v, s) + } +} + +// TestStale_GetLoadedValueServesSelfStale verifies the stale window applies +// uniformly regardless of how the value was written: a Get-loaded value, +// once expired, is served as Stale by TryGet for MaxStaleAge, exactly like +// a Set-loaded value. +// +// Regression test: only Set/Swap-created loadings used to carry a stale +// snapshot of themselves; Get-loaded values had none (their stale field is +// the previous generation's refresh companion), so TryGet reported Miss +// the instant they expired while a Get-driven refresh of the same state +// served the Stale. +func TestStale_GetLoadedValueServesSelfStale(t *testing.T) { + const ttl = 20 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(time.Hour)) + c.Get("k", func() (int, error) { return 1, nil }) + time.Sleep(2 * ttl) + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet on an expired Get-loaded value: (%d, %v), want (1, Stale)", v, s) + } +} + +// TestStale_ServesLatestGeneration verifies that once a refreshed value +// itself expires, every read serves that value as the stale — not the +// generation before it. +// +// Regression test: a Get-created loading's stale field is the previous +// generation's snapshot; TryGet used to serve it (v1) after the refreshed +// value (v2) expired, while a Get-driven refresh of the same state +// snapshotted and served v2. +func TestStale_ServesLatestGeneration(t *testing.T) { + const ttl = 30 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(time.Hour)) + c.Set("k", 1) + time.Sleep(ttl + 10*time.Millisecond) // v1 expired; within its stale window + + // Drive a refresh to v2; the driving Get returns the v1 stale while + // its (held-open) miss runs. + release := make(chan struct{}) + if v, _, s := c.Get("k", func() (int, error) { <-release; return 2, nil }); s != Stale || v != 1 { + t.Fatalf("refreshing Get: (%d, %v), want (1, Stale)", v, s) + } + close(release) + time.Sleep(ttl + 20*time.Millisecond) // v2 finalized, then expired + + if v, _, s := c.TryGet("k"); s != Stale || v != 2 { + t.Fatalf("TryGet after v2 expired: (%d, %v), want (2, Stale) — the freshest dead value", v, s) + } +} + +// TestMaxIdleAge_StaleWindowFollowsExtensions verifies that the stale +// window of an idle-extended entry anchors at the entry's actual death +// (its last extension lapsing), not at its initial expiry: derived from +// the expiry word, the window moves with every extension. +// +// Regression test: the self-stale used to be a snapshot frozen at +// creation, anchored at the initial expiry. Extensions moved only the +// expiry word, so an entry kept hot past initialExpiry+MaxStaleAge idled +// out with its stale window already closed: TryGet reported Miss the +// instant the entry expired while Get served the Stale. +func TestMaxIdleAge_StaleWindowFollowsExtensions(t *testing.T) { + const ( + ttl = 200 * time.Millisecond + age = 400 * time.Millisecond + ) + c := New[string, int](MaxAge(ttl), MaxIdleAge(ttl), MaxStaleAge(age)) + c.Set("k", 1) + + // Keep the entry hot well past the initial expiry + age (600ms after + // the store), so a creation-frozen window would be long dead. + for range 8 { // ~640ms of extensions + time.Sleep(80 * time.Millisecond) + if _, _, s := c.TryGet("k"); !s.IsHit() { + t.Fatal("entry should still be live while being extended") + } + } + time.Sleep(ttl + 100*time.Millisecond) // idle out: died ~100ms ago + + // ~100ms into the 400ms post-death window: still a Stale. + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet shortly after an idle-extended entry expired: (%d, %v), want (1, Stale)", v, s) + } + time.Sleep(age) // window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the stale window: %v, want Miss", s) + } +} + +// TestExpire_DoesNotResurrectDeadStale verifies that Expire on an entry +// whose value and stale window are both already dead is a no-op: the next +// Get drives a fresh load rather than serving the dead value. +// +// Regression test: Expire used to blindly store now()-1, moving an +// already-expired word forward; the next Get-driven refresh anchored a +// fresh stale window at the Expire and served the dead value as Stale for +// up to another MaxStaleAge. +func TestExpire_DoesNotResurrectDeadStale(t *testing.T) { + const ttl, age = 5 * time.Millisecond, 5 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(age)) + c.Set("k", 1) + time.Sleep(30 * time.Millisecond) // value expired AND stale window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: want Miss, got %v", s) + } + c.Expire("k") // invalidating a dead key must not revive anything + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get after expiring a dead entry: (%d, %v), want (2, Miss)", v, s) + } +} + +// TestExpire_StaleWindowAnchorsAtExpire verifies the uniform stale-window +// anchor for manually expired values: the window opens at the Expire and +// closes MaxStaleAge later, for TryGet and Get-driven refreshes alike. +// +// Regression test: TryGet used to serve a frozen creation-time snapshot +// whose window was anchored at the original expiry (an hour out here), +// outliving the documented Expire-anchored window. +func TestExpire_StaleWindowAnchorsAtExpire(t *testing.T) { + const age = 50 * time.Millisecond + c := New[string, int](MaxAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + c.Expire("k") + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet just after Expire: (%d, %v), want (1, Stale)", v, s) + } + time.Sleep(2 * age) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the post-Expire window: %v, want Miss", s) + } +} + // TestCompareAndSwapAgreesWithTryGet verifies that CompareAndSwap and // CompareAndDelete only match live values: expired entries and errored // entries are "not there" per TryGet, so comparing against them must fail. diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index 57c2e50..bace687 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -959,16 +959,18 @@ func TestExpire_NotLostToIdleExtension(t *testing.T) { // TestClean_ClaimWindowDoesNotResurrectDeadStale pins Clean's claim protocol // against racing Gets: Clean claims an entry's expiry word before // tombstoning, and a Get that lands between the claim and the tombstone -// snapshots that loading for a stale. The claim must not read as "born -// expired": entMaybeNewStale would re-anchor the certified-dead stale at now -// and the Get would serve a value past MaxAge+MaxStaleAge as a fresh Stale -// for up to another MaxStaleAge. +// snapshots that loading for a stale. The claimed word must read as "stale +// window certified closed" — never as something a stale window can be +// derived from — or the Get would serve a value past MaxAge+MaxStaleAge as +// a fresh Stale for up to another MaxStaleAge. // // The test performs Clean's claim by hand (same gate, same CAS) so the // window between Clean's two CASes is held open deterministically. // // Regression test: Clean used to claim with -1, the born-expired sentinel, -// which is exactly what newStale re-anchors. +// which newStale then re-anchored at now (born-expired words now carry +// their birth and anchor there, and a claimed word derives no window at +// all). func TestClean_ClaimWindowDoesNotResurrectDeadStale(t *testing.T) { c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) c.Set("k", 1) @@ -984,8 +986,9 @@ func TestClean_ClaimWindowDoesNotResurrectDeadStale(t *testing.T) { l := e.p.Load() expires := l.expires.Load() tn := now() - if expires <= 0 || tn <= satAdd(expires, int64(c.cfg.maxStaleAge)) || - (l.stale != nil && !l.stale.expired(tn)) { + if expired := expires != 0 && expires <= tn; !expired || + (l.stale != nil && !l.stale.expired(tn)) || + (l.err == nil && staleOpen(expires, tn, c.cfg.maxStaleAge)) { t.Fatalf("precondition: entry not Clean-eligible: expires=%d", expires) } if !l.expires.CompareAndSwap(expires, expiredClaimed) { @@ -1061,3 +1064,154 @@ func TestClean_ClaimWindowVsGetRace(t *testing.T) { wg.Wait() } } + +// TestExpire_DoesNotOverwriteCleanClaim pins Expire against Clean's claim +// window: once Clean has claimed an entry's expiry word (certifying the +// value expired and its stale window closed), a racing Expire must leave +// the claim in place — never replace it with a fresh timestamp that a Get +// in the window could re-derive a stale window from. Same hand-driven +// protocol as TestClean_ClaimWindowDoesNotResurrectDeadStale. +// +// Regression test: Expire used to blindly store now()-1 over any finalized +// word, including expiredClaimed; a Get between Clean's two CASes then +// served the certified-dead value as a fresh Stale. +func TestExpire_DoesNotOverwriteCleanClaim(t *testing.T) { + c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) + c.Set("k", 1) + time.Sleep(5 * time.Millisecond) // expired AND past the stale window + + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: entry should be fully dead, got %v", s) + } + + e := c.t.loadEntry("k") + l := e.p.Load() + expires := l.expires.Load() + if !l.expires.CompareAndSwap(expires, expiredClaimed) { // Clean's claim + t.Fatal("claim CAS failed") + } + + c.Expire("k") // lands between Clean's two CASes; must be a no-op + + if got := l.expires.Load(); got != expiredClaimed { + t.Fatalf("Expire overwrote Clean's claim: expires=%d, want expiredClaimed", got) + } + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get in the claim window: (%d, %v), want (2, Miss)", v, s) + } +} + +// TestSwap_OverFinalizedErroredWithStale is TestSwap_OverErroredWithStale +// with the race pinned the other way: the errored load has finalized by +// the time Swap displaces it. The classification must agree with TryGet: +// the valid stale, not the error, is the prior value. +// +// Regression test: the defer's finalized classification used to assign the +// stale and then fall through to overwrite it with (v, err, Hit); the +// sibling test only exercises the unfinalized branch (Swap usually beats +// the async setve), so the finalized path went unexercised. +func TestSwap_OverFinalizedErroredWithStale(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour)) + c.Set("k", 1) + c.Expire("k") + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + + // Wait for the async setve to finalize the errored loading. + e := c.t.loadEntry("k") + for l := e.p.Load(); l == nil || !l.finalized(); l = e.p.Load() { + runtime.Gosched() + } + + if v, err, s := c.TryGet("k"); s != Stale || v != 1 || err != nil { + t.Fatalf("TryGet: (%d, %v, %v), want (1, nil, Stale)", v, err, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale || old != 1 || oldErr != nil { + t.Fatalf("Swap: (%d, %v, %v), want (1, nil, Stale) to agree with TryGet", old, oldErr, oldS) + } +} + +// TestSwap_OverFinalizedErroredNoStale covers the remaining finalized +// classification arm: with no stale to return, an unexpired cached error +// is itself the prior result, and Swap reports it as a Hit carrying the +// error — agreeing with TryGet. +func TestSwap_OverFinalizedErroredNoStale(t *testing.T) { + c := New[string, int]() // errors cache forever by default + // The driving Get waits for its load, so the error is finalized here. + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + + if _, err, s := c.TryGet("k"); err == nil || !s.IsHit() { + t.Fatalf("TryGet: err=%v s=%v, want the cached error as a Hit", err, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if old != 0 || oldErr == nil || !oldS.IsHit() { + t.Fatalf("Swap: (%d, %v, %v), want (0, boom, Hit) to agree with TryGet", old, oldErr, oldS) + } +} + +// TestDelete_ConcurrentlyFinalizedLoadReportsHit pins Delete's documented +// classification race: Delete samples the clock before tombstoning, but +// reads the loading's finalized state afterwards, so a load finalizing +// between the two is reported as the removed value (a Hit) even though no +// read ever observed it cached. This is Delete's exact code sequence with +// the window held open; the returned triple is best effort there (see +// Delete's doc), and the cache state itself is unaffected. +func TestDelete_ConcurrentlyFinalizedLoadReportsHit(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + missRelease := make(chan struct{}) + getDone := make(chan struct{}) + go func() { + defer close(getDone) + c.Get("k", func() (int, error) { <-missRelease; return 7, nil }) + }() + var e *ent[string, int] + for e = c.t.loadEntry("k"); e == nil; e = c.t.loadEntry("k") { + runtime.Gosched() + } + + // Delete's sequence (see Cache.Delete), paused mid-window: + n := now() + was := entDel(e) + c.t.deleteEntryIf("k", entDead[string, int]) + close(missRelease) // the miss finalizes the already-detached loading + <-getDone + + if v, _, s := loadingTryGet(was, n, 0, c.cfg.maxStaleAge); !s.IsHit() || v != 7 { + t.Fatalf("classifying the concurrently finalized loading: (%d, %v), want (7, Hit) per Delete's documented race", v, s) + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("the key must be gone regardless: %v, want Miss", s) + } +} + +// TestGet_CollapsedErrorWaitersRedrive pins the documented MaxErrorAge(0) +// collapsing behavior (see MaxErrorAge): Gets that collapsed onto a load +// that then errors do not share the error; each waiter re-drives its own +// load in turn, so one erroring wave of N collapsed Gets issues N loads. +func TestGet_CollapsedErrorWaitersRedrive(t *testing.T) { + c := New[string, int](MaxErrorAge(0)) + var calls atomic.Int32 + release := make(chan struct{}) + started := make(chan struct{}) + var once sync.Once + miss := func() (int, error) { + if calls.Add(1) == 1 { + once.Do(func() { close(started) }) + <-release + } + return 0, errors.New("boom") + } + const waiters = 8 + var wg sync.WaitGroup + wg.Add(waiters) + for range waiters { + go func() { defer wg.Done(); c.Get("k", miss) }() + } + <-started + time.Sleep(20 * time.Millisecond) // let the others queue on the leader + close(release) + wg.Wait() + if n := calls.Load(); n != waiters { + t.Fatalf("miss ran %d times for %d collapsed Gets; with errors uncached, every Get drives exactly one load", n, waiters) + } +} From 1afccb4d7abfab9f61036c105be621cdaec80973 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 17:29:18 -0600 Subject: [PATCH 13/21] cache: coherent (expiry, clock) read pairs; one shared classification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth audit round (the read-time stale window machinery), one Low: - A read assembled its classification from two separate loads: the expires word, then the clock. A reader descheduled between them while an idle extension landed classified the dead pre-extension word at a fresh clock — TryGet reported Stale (with MaxStaleAge) or Miss (without) for an entry that was live at every instant of its call, and a later TryGet returned Hit with no intervening write, a history no sequential order explains. Get healed itself by re-deriving in its slow loop; TryGet surfaced it. expiresNow now pairs the word with the clock coherently: load, sample, confirm the word unchanged, re-deriving on movement. Each retry re-loads before re-sampling, preserving the ordering that classifies a concurrent Expire as expired rather than extending over it. Both readers use it; Range's and Delete's caller-pinned clocks predate the word load, so that inverted pairing is already conservative. The repros hold the deschedule window open via expiresPairingHook, an unexported nil-by-default test hook (the trie hashFn precedent), and were confirmed red against the pre-fix code. - The finalized classification was hand-mirrored across loadingTryGet, entGet, and Swap's displaced-value report — a divergence class that already bit once. classifyFinalized is now the one shared ladder; entGet's waited-courtesy precheck stays at its call site, and the idle-extension epilogue keys off the classified state. Doc precision: an Expire racing the entry's natural death can land just after it and trim (never grow) the dead value's remaining stale window; non-positive MaxIdleAge is ignored entirely. Plus five pins for cells the suite did not assert: Swap displacing a born-expired value (both sides of the birth-anchored window), Expire on an expired-but-in-window word (byte-identical no-op, positive and negated-birth words), Clean keeping an expired error whose companion is still servable, exact-nanosecond window boundaries across staleOpen, newStale, and stale.expired, and the safe (backward) direction of the paired read. --- cache/cache.go | 191 +++++++++++++++++++------------------- cache/cache_test.go | 127 +++++++++++++++++++++++++ cache/concurrency_test.go | 144 ++++++++++++++++++++++++++++ 3 files changed, 367 insertions(+), 95 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 1262ad3..88e09be 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -246,9 +246,11 @@ func MaxErrorAge(age time.Duration) Opt { // MaxIdleAge opts in to extending an entry's expiry on each successful // access. Each time Get or TryGet returns a live Hit without an error, the // entry's expiry is reset to now + age. If MaxAge is not set, the idle -// age is also used as the initial TTL. The reset applies in both -// directions: with MaxIdleAge smaller than MaxAge, an access shortens the -// entry's remaining life from the MaxAge-given expiry to now + age. +// age is also used as the initial TTL. A non-positive age is ignored +// entirely: no extension occurs, and no initial TTL is derived from it. +// The reset applies in both directions: with MaxIdleAge smaller than +// MaxAge, an access shortens the entry's remaining life from the +// MaxAge-given expiry to now + age. // // The reset is best effort under races: a concurrent Expire, or a Clean // that has already deemed the entry expired, wins over an in-flight @@ -479,7 +481,9 @@ func (c *Cache[K, V]) Delete(k K) (v V, err error, _ KeyState) { // complete with its normal TTL and is not canceled or shortened. Expiring an // already-expired entry is also a no-op: the expiry is never moved forward, // so a dead value's stale window cannot be re-anchored or revived, and a -// concurrent Clean's claim is never overwritten. +// concurrent Clean's claim is never overwritten. (An Expire that races the +// entry's natural death can land just after it and trim the dead value's +// remaining stale window; the window only ever shrinks, never grows.) func (c *Cache[K, V]) Expire(k K) { e := c.t.loadEntry(k) if e == nil { @@ -651,28 +655,10 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { } was.mu.Unlock() } - // Mirror loadingTryGet's classification so the displaced value is - // reported exactly as a TryGet at wasN would have reported it: a - // valid companion stale outranks an error, an unexpired error is - // itself the cached result, and an expired err-free value is - // still the prior value (Stale) while its derived window is open. - expires := was.expires.Load() - expired := expires != 0 && expires <= n - if was.err != nil { - if was.stale != nil && !was.stale.expired(n) { - old, oldState = was.stale.v, Stale - } else if !expired { - old, oldErr, oldState = was.v, was.err, Hit - } - return - } - if expired { - if staleOpen(expires, n, c.cfg.maxStaleAge) { - old, oldState = was.v, Stale - } - return - } - old, oldErr, oldState = was.v, was.err, Hit + // The displaced value is reported exactly as a TryGet at wasN + // would have reported it: classifyFinalized is the one + // classification every read shares. + old, oldErr, oldState = classifyFinalized(was, was.expires.Load(), n, c.cfg.maxStaleAge) }() // Fast path: if the entry already exists with a live value, CAS the @@ -943,6 +929,63 @@ func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { return &stale[V]{v, satAdd(expires, int64(age))} } +// expiresPairingHook, if non-nil, runs in expiresNow between the expiry +// load and the clock sample, so tests can deterministically hold a reader +// descheduled in that window. Never set in production code. +var expiresPairingHook func() + +// expiresNow returns the loading's expiry word and a clock sample forming a +// coherent pair. The word is loaded before the clock is sampled: Expire +// CASes in now()-1 from its own clock, which can be ahead of a clock +// sampled before our load, so sampling after the load guarantees a +// concurrent Expire is classified as expired rather than extended over. The +// word is then confirmed after the sample: an idle extension landing +// between the two would otherwise pair the dead pre-extension word with a +// fresh clock, misreporting an entry that is live for this entire call as +// Stale or Miss. Each retry re-loads before re-sampling, preserving the +// Expire ordering. +func (l *loading[V]) expiresNow() (expires, n int64) { + expires = l.expires.Load() + if expiresPairingHook != nil { + expiresPairingHook() + } + for { + n = now() + if again := l.expires.Load(); again != expires { + expires = again + continue + } + return expires, n + } +} + +// classifyFinalized reports a finalized loading exactly as a read at clock +// n does: a valid companion stale (the previous generation) outranks an +// error; with no companion, an unexpired error is itself the cached result +// and a dead error is a miss; an expired err-free value is served as Stale +// while its derived window is open (see staleOpen) and is a miss after. +// entGet, loadingTryGet, and Swap's displaced-value report all share this +// classification, so every read and every report agree by construction. +func classifyFinalized[V any](l *loading[V], expires, n int64, staleAge time.Duration) (v V, err error, state KeyState) { + expired := expires != 0 && expires <= n + if l.err != nil { + if l.stale != nil && !l.stale.expired(n) { + return l.stale.v, nil, Stale + } + if !expired { + return l.v, l.err, Hit + } + return v, err, state + } + if expired { + if staleOpen(expires, n, staleAge) { + return l.v, nil, Stale + } + return v, err, state + } + return l.v, l.err, Hit +} + // entGet returns the value, the stale value, or — after waiting on an // in-flight load — whatever the load produced. When we waited, expiry alone // does not force a miss: Get must hand its caller something even if the @@ -962,47 +1005,25 @@ func entGet[K comparable, V any](e *ent[K, V], extend, staleAge time.Duration) ( waited = true } - // The expiry must be loaded before the clock is sampled: Expire CASes - // in now()-1 from its own clock, which can be ahead of a clock sampled - // before our load; sampling after the load guarantees a concurrent - // Expire is classified as expired here rather than extended over. - expires := l.expires.Load() - n := now() + expires, n := l.expiresNow() expired := expires != 0 && expires <= n - if l.err != nil { - // Errored: a valid companion stale (the previous generation) - // outranks the error; with no companion, the error itself is the - // cached result while it lives, and a dead error is a miss. - if l.stale != nil && !l.stale.expired(n) { - return l.stale.v, nil, Stale - } - if !expired { - return l.v, l.err, Hit - } - return v, err, state - } - if expired { - // If we waited, the load could finalize already expired (MaxAge(0) + if waited && l.err == nil && expired { + // We waited and the load finalized already expired (MaxAge(0) // collapsing, or a TTL shorter than the load itself): hand back - // the now-expired value as a courtesy rather than re-drive. - // Otherwise the expired value is served as Stale while its - // derived window is open (see staleOpen), and is a miss after. - if waited { - return l.v, nil, Hit - } - if staleOpen(expires, n, staleAge) { - return l.v, nil, Stale - } - return v, err, state + // the now-expired value as a courtesy rather than re-drive. The + // courtesy is not an idle-extending access; the value is + // returned, not resurrected. + return l.v, nil, Hit } - if extend > 0 { + v, err, state = classifyFinalized(l, expires, n, staleAge) + if extend > 0 && state == Hit && l.err == nil { // Extend the idle expiry via CAS against the expiry we evaluated // above so that a concurrent Expire (or Swap finalization, or a // Clean eviction) is not silently overwritten; if expires - // changed, the other writer wins. This path is structurally - // unexpired at n — a waited-on load that finalized already - // expired was returned above as a courtesy, not resurrected — - // but n may have gone stale if we were descheduled since it was + // changed, the other writer wins. An err-free Hit is structurally + // unexpired at n (a waited-on load that finalized already expired + // was returned above as a courtesy, never reaching here), but n + // may have gone stale if we were descheduled since it was // sampled: a CAS landing after the expiry passed would revive an // entry that concurrent readers may have already reported as // Miss. Re-sample the clock immediately before publishing; the @@ -1012,7 +1033,7 @@ func entGet[K comparable, V any](e *ent[K, V], extend, staleAge time.Duration) ( l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) } } - return l.v, l.err, Hit + return v, err, state } func entTryGet[K comparable, V any](e *ent[K, V], n64 int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { @@ -1043,47 +1064,27 @@ func loadingTryGet[V any](l *loading[V], n64 int64, extend, staleAge time.Durati return v, err, state } - // The expiry must be loaded before the clock is sampled (when we are - // the one sampling it); see the matching comment in entGet. A caller- - // provided n64 (Range's batch timestamp, Delete's pre-removal clock) - // never extends, so the ordering does not matter there. - expires := l.expires.Load() + var expires int64 if n64 == 0 { - n64 = now() + expires, n64 = l.expiresNow() + } else { + // A caller-provided n64 (Range's batch timestamp, Delete's + // pre-removal clock) predates this load of the word, so the + // stale-word hazard expiresNow guards against cannot arise; the + // caller pins its snapshot to n64 and never extends. + expires = l.expires.Load() } n := n64 - expired := expires != 0 && expires <= n - if l.err != nil { - // Errored: a valid companion stale (the previous generation) - // outranks the error; with no companion, the error itself is the - // cached result while it lives, and a dead error is a miss. - if l.stale != nil && !l.stale.expired(n) { - return l.stale.v, nil, Stale - } - if expired { - return v, err, state - } - return l.v, l.err, Hit - } - if expired { - // An expired value is served as Stale while its derived window - // is open (see staleOpen), and is a miss after. - if staleOpen(expires, n, staleAge) { - return l.v, nil, Stale - } - return v, err, state - } - if extend > 0 { + v, err, state = classifyFinalized(l, expires, n, staleAge) + if extend > 0 && state == Hit && l.err == nil { // CAS so a concurrent Expire is not silently overwritten; see the - // matching comment in entGet. This path is structurally unexpired - // at n (the branch above returns) — but n may have gone stale if - // we were descheduled since it was sampled, so re-sample the - // clock immediately before publishing, as in entGet. + // matching comment in entGet, including the pre-publish clock + // re-sample. if now() < expires { l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) } } - return l.v, l.err, Hit + return v, err, state } ////////// diff --git a/cache/cache_test.go b/cache/cache_test.go index 6832a0a..6aabc65 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -3,6 +3,7 @@ package cache import ( "errors" "math" + "runtime" "sync" "sync/atomic" "testing" @@ -752,6 +753,132 @@ func TestExpire_StaleWindowAnchorsAtExpire(t *testing.T) { } } +// TestExpire_MidWindowDoesNotReanchor verifies Expire is a no-op on an +// entry that is expired but still inside its stale window: the expiry word +// (the window's anchor) must be byte-identical after the call. +// TestExpire_DoesNotResurrectDeadStale covers the window-already-closed +// case; here a forward re-anchor would extend a dead value's servable life +// mid-window. +func TestExpire_MidWindowDoesNotReanchor(t *testing.T) { + t.Run("ttl_expired", func(t *testing.T) { + c := New[string, int](MaxAge(20*time.Millisecond), MaxStaleAge(time.Hour)) + c.Set("k", 1) + time.Sleep(40 * time.Millisecond) // expired; window open for ~1h + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + c.Expire("k") + if w1 := l.expires.Load(); w1 != w0 { + t.Fatalf("Expire moved an already-expired word: %d -> %d (re-anchors the stale window)", w0, w1) + } + }) + t.Run("born_expired", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) // word carries the negated birth + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + if w0 >= 0 { + t.Fatalf("precondition: want a negated-birth word, got %d", w0) + } + c.Expire("k") + if w1 := l.expires.Load(); w1 != w0 { + t.Fatalf("Expire destroyed the birth anchor: %d -> %d", w0, w1) + } + }) +} + +// TestClean_KeepsExpiredErrorWithValidCompanion pins Clean's gate for +// errored entries: an expired error whose refresh companion (the previous +// generation) is still inside its window is visible to reads — TryGet +// serves the companion — so Clean must not reclaim it. +func TestClean_KeepsExpiredErrorWithValidCompanion(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour), MaxErrorAge(20*time.Millisecond)) + c.Set("k", 1) + c.Expire("k") + // Drive a refresh that errors; the new loading carries companion=1. + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + e := c.t.loadEntry("k") + for l := e.p.Load(); l == nil || !l.finalized(); l = e.p.Load() { + runtime.Gosched() + } + time.Sleep(40 * time.Millisecond) // error now expired; companion valid ~1h + + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("precondition: TryGet=(%d,%v), want (1, Stale) via the companion", v, s) + } + c.Clean() + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("Clean reclaimed an entry whose companion was still servable: TryGet=(%d,%v)", v, s) + } + if n := trieEntries(c); n != 1 { + t.Fatalf("trie entries = %d, want 1", n) + } +} + +// TestStale_WindowBoundaries drives the exact-nanosecond boundaries of the +// derived stale window through loadingTryGet with hand-picked clocks (the +// n64 parameter pins the clock deterministically, exactly as Range and +// Delete do) and checks the window definitions agree bit-for-bit across +// the read path (staleOpen), the refresh companion (newStale), and +// stale.expired: the window is [expiry, expiry+age), half-open at every +// site, for positive, negated-birth, and claimed words. +func TestStale_WindowBoundaries(t *testing.T) { + const age = 100 * time.Millisecond + a := int64(age) + c := New[string, int](MaxAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + l := c.t.loadEntry("k").p.Load() + + check := func(n int64, wantS KeyState, wantV int, what string) { + t.Helper() + v, _, s := loadingTryGet(l, n, 0, age) + if s != wantS || (wantS != Miss && v != wantV) { + t.Fatalf("%s: loadingTryGet=(%d,%v), want (%d,%v)", what, v, s, wantV, wantS) + } + } + + // Positive word: expiry E, window [E, E+age). + E := now() - 1 + l.expires.Store(E) + check(E-1, Hit, 1, "one ns before expiry") + check(E, Stale, 1, "the expiry instant itself") + check(E+a-1, Stale, 1, "last open ns of the window") + check(E+a, Miss, 0, "the window-close instant") + + if st := newStale(1, E, age); st.expires != E+a { + t.Fatalf("companion window end %d != read-path window end %d", st.expires, E+a) + } else if st.expired(E+a-1) || !st.expired(E+a) { + t.Fatal("companion boundary disagrees with the read-path boundary") + } + if !staleOpen(E, E+a-1, age) || staleOpen(E, E+a, age) { + t.Fatal("staleOpen boundary inconsistent with itself") + } + + // Negated-birth word: anchor B, window [B, B+age). + B := now() + l.expires.Store(-B) + check(B+a-1, Stale, 1, "born-expired: last open ns") + check(B+a, Miss, 0, "born-expired: window-close instant") + if st := newStale(1, -B, age); st.expires != B+a { + t.Fatalf("born-expired companion end %d, want %d", st.expires, B+a) + } + + // Claimed word: no window, ever; and no companion may be derived. + l.expires.Store(expiredClaimed) + check(B, Miss, 0, "claimed word") + if staleOpen(expiredClaimed, 1, age) { + t.Fatal("staleOpen derived a window from a claimed word") + } + if st := entMaybeNewStale(c.t.loadEntry("k"), age); st != nil { + t.Fatalf("entMaybeNewStale snapshotted a claimed loading: %+v", st) + } +} + // TestCompareAndSwapAgreesWithTryGet verifies that CompareAndSwap and // CompareAndDelete only match live values: expired entries and errored // entries are "not there" per TryGet, so comparing against them must fail. diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index bace687..2c691ad 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -1215,3 +1215,147 @@ func TestGet_CollapsedErrorWaitersRedrive(t *testing.T) { t.Fatalf("miss ran %d times for %d collapsed Gets; with errors uncached, every Get drives exactly one load", n, waiters) } } + +// armExpiresPairingHook installs expiresPairingHook such that exactly the +// first read to pass through it blocks until release is closed; all later +// reads pass straight through. The hook is cleared when the test ends. +func armExpiresPairingHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + entered = make(chan struct{}) + release = make(chan struct{}) + var fired atomic.Bool + expiresPairingHook = func() { + if fired.CompareAndSwap(false, true) { + close(entered) + <-release + } + } + t.Cleanup(func() { expiresPairingHook = nil }) + return entered, release +} + +// TestTryGet_PairedReadVsIdleExtension pins the coherence of the (expiry +// word, clock) pair a read classifies with: a reader descheduled between +// loading the word and sampling the clock, with an idle extension landing +// in that window, would otherwise classify the dead pre-extension word at +// a fresh clock and report Stale (with MaxStaleAge) or Miss (without) for +// an entry that was live at every instant of its call — and a later TryGet +// returns Hit with no intervening write, a history no sequential order +// explains. expiresNow re-confirms the word after sampling the clock, so +// the read classifies the extended (live) word instead. +// +// The hook holds the deschedule window open deterministically, like the +// hand-driven claim-window tests above. +// +// Regression test: loadingTryGet and entGet used to classify whatever word +// they loaded first at whatever clock they sampled next. +func TestTryGet_PairedReadVsIdleExtension(t *testing.T) { + const ttl = 100 * time.Millisecond + for _, tt := range []struct { + name string + opts []Opt + }{ + {"no_stale_age", []Opt{MaxAge(ttl), MaxIdleAge(time.Hour)}}, + {"with_stale_age", []Opt{MaxAge(ttl), MaxIdleAge(time.Hour), MaxStaleAge(time.Hour)}}, + } { + t.Run(tt.name, func(t *testing.T) { + c := New[string, int](tt.opts...) + c.Set("k", 1) + + entered, release := armExpiresPairingHook(t) + type res struct { + v int + s KeyState + } + r1c := make(chan res, 1) + go func() { + v, _, s := c.TryGet("k") // loads the initial expiry, parks in the hook + r1c <- res{v, s} + }() + <-entered + + // A second TryGet (the hook is one-shot) hits and extends the + // entry to ~now+1h; the entry is now live well past the + // original expiry. + if v, _, s := c.TryGet("k"); s != Hit || v != 1 { + t.Fatalf("extending TryGet: (%d, %v), want (1, Hit)", v, s) + } + // Let real time pass the original expiry while the first + // reader is still parked between its two loads. + time.Sleep(ttl + 50*time.Millisecond) + close(release) + r1 := <-r1c + + if v, _, s := c.TryGet("k"); s != Hit || v != 1 { + t.Fatalf("entry must still be live after the extension: (%d, %v)", v, s) + } + if r1.s != Hit || r1.v != 1 { + t.Fatalf("paired read returned (%d, %v) for an entry that was live throughout its call, want (1, Hit)", r1.v, r1.s) + } + }) + } +} + +// TestTryGet_PairedReadVsExpire pins the safe direction of the same window: +// otherwise the word only ever moves backward (Expire, Clean's claim), and +// a parked reader lands on one side of the race or the other — Hit per the +// pre-Expire live word (its call overlaps the Expire, so ordering the read +// first explains the history) or Miss per the expired word. Whichever side +// it lands on, no stale is conjured for an entry with no stale configured. +func TestTryGet_PairedReadVsExpire(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + + entered, release := armExpiresPairingHook(t) + type res struct { + v int + s KeyState + } + r1c := make(chan res, 1) + go func() { + v, _, s := c.TryGet("k") + r1c <- res{v, s} + }() + <-entered + + c.Expire("k") // moves the word backward while the reader is parked + close(release) + r1 := <-r1c + + if r1.s == Stale { + t.Fatalf("no stale is configured; got (%d, %v)", r1.v, r1.s) + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("after Expire: %v, want Miss", s) + } +} + +// TestSwap_OverBornExpired pins the born-expired classification cell of +// Swap's displaced-value report against TryGet: a displaced born-expired +// value (negated-birth expiry word) is the prior value (Stale) while its +// birth-anchored window is open, and Miss after — exactly as TryGet +// reports the same state. +func TestSwap_OverBornExpired(t *testing.T) { + t.Run("window_open", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet: (%d, %v), want (1, Stale)", v, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale || old != 1 || oldErr != nil { + t.Fatalf("Swap: (%d, %v, %v), want (1, nil, Stale) to agree with TryGet", old, oldErr, oldS) + } + }) + t.Run("window_closed", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(30*time.Millisecond)) + c.Set("k", 1) + time.Sleep(60 * time.Millisecond) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet: %v, want Miss", s) + } + if _, _, oldS := c.Swap("k", 2); oldS != Miss { + t.Fatalf("Swap: %v, want Miss to agree with TryGet", oldS) + } + }) +} From 0617c23bd02e945fabd46edcafb97a3220ad20b3 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 18:32:09 -0600 Subject: [PATCH 14/21] cache: re-validate liveness before publishing a CompareAnd{Swap,Delete} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sixth audit round (full re-read of cache.go and trie.go plus every test file, all six scenario classes re-traced), one Low: - tryCAS sampled the matched value's liveness once, then allocated the replacement loading, then published with the pointer CAS — the whole allocation (the widest deschedule risk in the call) sat between the liveness sample and the publish. A CAS caller parked there while the value's TTL lapsed still published: TryGet had already certified Miss, and no sequential order explains TryGet=Miss followed by CAS=true (CAS-first requires the TryGet to see the freshly installed value; TryGet-first requires the CAS to fail on an expired match). The window cannot be closed outright: a successful swap takes effect at the pointer CAS, and the clock cannot be read at that exact instant — pre-claiming the expiry word would instead delay observable expiry, and post-CAS rollback can race a reader that already observed the new value. So this follows the MaxIdleAge precedent: tryCAS now re-validates immediately before every CAS attempt, shrinking the window from "alloc + deschedule" to a few instructions, and the residue is documented on CompareAndSwap and CompareAndDelete (the mismatch direction stays exact). The repro holds the deschedule open via casPublishHook (nil-by-default, the expiresPairingHook precedent) and was confirmed red against the pre-fix ordering for both CAS and CAD. The linearizability checker deliberately runs an expiry-free config (expiry is a clock transition, not an operation), so the hook pin is the vehicle for this finding, as in rounds three through five. Doc accuracy: AutoCleanInterval now states that with a negative MaxStaleAge the goroutine is not started at all (Clean would be a no-op; pinned by TestAutoClean_DisabledWhenStaleAgeNegative); Item's CompareAndSwap/CompareAndDelete point at the Cache caveat. Everything else re-verified clean this round: tombstone terminality under the bucket lock, the claim sentinel against every expires-word consumer, derived stale windows being absolute intervals (a late snapshot cannot extend one), prune lock ordering (child linked implies parent not dead), Get's pin-and-re-derive slow path, and the WaitGroup/mutex finalize-once protocol. --- cache/cache.go | 36 +++++++++++++++++++-- cache/concurrency_test.go | 68 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 88e09be..291ce9f 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -272,6 +272,9 @@ func MaxIdleAge(age time.Duration) Opt { // // The goroutine holds a reference to the cache, so a cache that started // autocleaning is not garbage collected until StopAutoClean is called. +// +// If MaxStaleAge is negative (stales kept forever), Clean is a no-op and +// the goroutine is not started at all. func AutoCleanInterval(interval time.Duration) Opt { return opt{fn: func(c *cfg) { c.autoCleanInterval = interval }} } @@ -729,6 +732,9 @@ func (c *Cache[K, V]) Set(k K, v V) { // loading without an error, is not expired, and is equal to old. The type V // must be comparable. Stale values are not considered: only the live value // is compared against. +// +// The expiry check is best effort: a value whose TTL lapses while the swap +// is in flight may still be swapped. func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { e := c.t.loadEntry(k) if e == nil { @@ -741,6 +747,9 @@ func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { // without an error, is not expired, and is equal to old. The type V must be // comparable. Stale values are not considered: only the live value is // compared against. +// +// The expiry check is best effort: a value whose TTL lapses while the +// delete is in flight may still be deleted. func (c *Cache[K, V]) CompareAndDelete(k K, old V) bool { e := c.t.loadEntry(k) if e == nil { @@ -764,14 +773,23 @@ func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { if useNew { l2 = c.finalizedLoading(new) } + if casPublishHook != nil { + casPublishHook() + } for { + // Re-validate liveness immediately before publishing: the + // allocation above is the widest deschedule risk in this call, and + // a value whose TTL lapsed while we were parked there must not be + // matched — TryGet already reports it as Miss. The handful of + // instructions between this check and the CAS remain best effort + // (see CompareAndSwap). + if !casMatches(l, old) { + return false + } if e.p.CompareAndSwap(l, l2) { return true } l = e.p.Load() - if !casMatches(l, old) { - return false - } } } @@ -781,6 +799,12 @@ func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { // errored entry is not a value that can be compared against (an errored // loading's v is whatever the miss function returned beside the error, // which was never cached as a value). +// +// The liveness sample cannot be made atomic with the publishing CAS: a +// successful swap takes effect at the pointer CAS, and the clock cannot be +// read at that exact instant. tryCAS narrows the gap by re-validating +// immediately before each CAS attempt; the instructions-wide residue is +// documented on CompareAndSwap and CompareAndDelete. func casMatches[V any](l *loading[V], old V) bool { return l != nil && l.finalized() && l.err == nil && any(l.v) == any(old) && !l.expired(now()) @@ -934,6 +958,12 @@ func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { // descheduled in that window. Never set in production code. var expiresPairingHook func() +// casPublishHook, if non-nil, runs in tryCAS between the replacement +// loading's allocation and the pre-publish liveness re-validation, so tests +// can deterministically hold a CAS caller descheduled in that window. Never +// set in production code. +var casPublishHook func() + // expiresNow returns the loading's expiry word and a clock sample forming a // coherent pair. The word is loaded before the clock is sampled: Expire // CASes in now()-1 from its own clock, which can be ahead of a clock diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index 2c691ad..762aedb 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -1234,6 +1234,74 @@ func armExpiresPairingHook(t *testing.T) (entered, release chan struct{}) { return entered, release } +// armCASPublishHook installs casPublishHook such that exactly the first +// tryCAS to pass through it blocks until release is closed; all later calls +// pass straight through. The hook is cleared when the test ends. +func armCASPublishHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + entered = make(chan struct{}) + release = make(chan struct{}) + var fired atomic.Bool + casPublishHook = func() { + if fired.CompareAndSwap(false, true) { + close(entered) + <-release + } + } + t.Cleanup(func() { casPublishHook = nil }) + return entered, release +} + +// TestTryCAS_RevalidatesLivenessBeforePublish pins tryCAS's pre-publish +// re-validation: a CAS caller that matched a live value and was then +// descheduled (the replacement's allocation is the widest such window in +// the call) while the value's TTL lapsed must not publish — the value it +// matched is one TryGet already certifies as Miss, and no sequential order +// explains TryGet=Miss followed by CAS=true. The residual instructions +// between the re-validation and the pointer CAS remain best effort (see +// CompareAndSwap's doc). +// +// Regression test: tryCAS used to sample liveness only before the +// allocation, so the entire alloc + deschedule window sat between the +// liveness sample and the publish. +func TestTryCAS_RevalidatesLivenessBeforePublish(t *testing.T) { + const ttl = 20 * time.Millisecond + for _, m := range []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } { + t.Run(m.name, func(t *testing.T) { + c := New[string, int](MaxAge(ttl)) + c.Set("k", 1) + + entered, release := armCASPublishHook(t) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + <-entered + + // Let the matched value's TTL lapse while the caller is parked + // between its liveness sample and its publish. + for { + if _, _, s := c.TryGet("k"); s.IsMiss() { + break + } + time.Sleep(time.Millisecond) + } + close(release) + + if <-res { + t.Fatal("published against a value that expired mid-call (TryGet had already certified Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + }) + } +} + // TestTryGet_PairedReadVsIdleExtension pins the coherence of the (expiry // word, clock) pair a read classifies with: a reader descheduled between // loading the word and sampling the clock, with an idle extension landing From e8c1448c2f5cf023c4a25625fa9485f8562ff14a Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 21:56:06 -0600 Subject: [PATCH 15/21] cache: complete the CompareAnd{Swap,Delete} doc caveats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seventh audit round (full re-read of cache.go, trie.go, and every test file; all six scenario classes re-traced), two Lows, both doc accuracy on the same caveat surface: - The sixth round's commit message claims Item's CompareAndSwap and CompareAndDelete "point at the Cache caveat", but that change never made it into the diff: the Item docs carried neither the stale-values-not-considered sentence nor the best-effort expiry caveat. Both now point at the Cache method docs, following the Item.Get "See Cache.Get" precedent. - The Cache caveat described the best-effort residue as "a value whose TTL lapses while the swap is in flight"; the same instructions-wide window also lets a concurrent Expire's kill slip through, and that direction is otherwise unexplainable (Expire-first requires the CAS to fail on the dead match; CAS-first requires the Expire to kill the swapped-in value, which a later TryGet=Hit contradicts). The window is irreducible for Expire exactly as for TTL lapse — Expire writes the expiry word while the publish is a pointer CAS, and the sixth round already weighed and rejected pre-claiming the word and post-CAS rollback — so the caveat now says expiry generally: TTL or Expire. The wide (allocation) side of the Expire direction is exact, not best effort: the pre-publish re-validation loads the expiry word fresh, so an Expire landing while the caller is parked there is caught. Pinned by TestTryCAS_RevalidationCatchesExpire via casPublishHook (the TTL sibling only lapses a deadline), confirmed red against the pre-revalidation ordering for both CAS and CAD. Everything else re-verified clean this round: the claim sentinel against Swap/Delete/Expire/extension (including a benign by-value ABA on the expires word: a claim landing after extend-then-Expire returns the word to its evaluated value is semantically identical to the no-race claim), tombstone-shell unlink-then-recreate under every writer path (never two live entries for one key), walk's single-visit-per-key claim under expand and prune, Clear against in-flight writers on the old root, the error/companion classification ladder, and Clean's conservative companion gate for err-free values (provably never outlives the derived self-window gate). --- cache/cache.go | 16 ++++++++++------ cache/concurrency_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 291ce9f..e58f8bf 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -733,8 +733,9 @@ func (c *Cache[K, V]) Set(k K, v V) { // must be comparable. Stale values are not considered: only the live value // is compared against. // -// The expiry check is best effort: a value whose TTL lapses while the swap -// is in flight may still be swapped. +// The expiry check is best effort: a value that expires while the swap is +// in flight — its TTL lapsing or a concurrent Expire landing — may still +// be swapped. func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { e := c.t.loadEntry(k) if e == nil { @@ -748,8 +749,9 @@ func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { // comparable. Stale values are not considered: only the live value is // compared against. // -// The expiry check is best effort: a value whose TTL lapses while the -// delete is in flight may still be deleted. +// The expiry check is best effort: a value that expires while the delete +// is in flight — its TTL lapsing or a concurrent Expire landing — may +// still be deleted. func (c *Cache[K, V]) CompareAndDelete(k K, old V) bool { e := c.t.loadEntry(k) if e == nil { @@ -1192,14 +1194,16 @@ func (i *Item[V]) Swap(v V) (old V, oldErr error, oldState KeyState) { // CompareAndSwap swaps the old and new values if the value has finished // loading without an error, is not expired, and is equal to old. The type V -// must be comparable. +// must be comparable. See Cache.CompareAndSwap for the stale-handling and +// best-effort expiry caveats. func (i *Item[V]) CompareAndSwap(old, new V) (swapped bool) { return i.c.CompareAndSwap(struct{}{}, old, new) } // CompareAndDelete deletes the item if the value has finished loading // without an error, is not expired, and is equal to old. The type V must be -// comparable. +// comparable. See Cache.CompareAndDelete for the stale-handling and +// best-effort expiry caveats. func (i *Item[V]) CompareAndDelete(old V) (deleted bool) { return i.c.CompareAndDelete(struct{}{}, old) } diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index 762aedb..e193d26 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -1302,6 +1302,44 @@ func TestTryCAS_RevalidatesLivenessBeforePublish(t *testing.T) { } } +// TestTryCAS_RevalidationCatchesExpire extends the pre-publish re-validation +// pin to manual expiry: a CAS caller that matched a live value and was then +// parked at the replacement's allocation while a concurrent Expire killed +// the value must not publish — Expire's word write is visible to the +// re-validation's fresh liveness check, the same way a lapsed TTL is. Only +// the instructions between the re-validation and the pointer CAS remain +// best effort for Expire, exactly as for a lapsing TTL (see +// CompareAndSwap's doc). +func TestTryCAS_RevalidationCatchesExpire(t *testing.T) { + for _, m := range []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } { + t.Run(m.name, func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + + entered, release := armCASPublishHook(t) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + <-entered + + c.Expire("k") // kills the matched value while the caller is parked + close(release) + + if <-res { + t.Fatal("published against a value a concurrent Expire had already killed (TryGet certifies Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + }) + } +} + // TestTryGet_PairedReadVsIdleExtension pins the coherence of the (expiry // word, clock) pair a read classifies with: a reader descheduled between // loading the word and sampling the clock, with an idle extension landing From ac4d8d3ef01e35025bcfef6824da13527fae8fdc Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 22:18:05 -0600 Subject: [PATCH 16/21] cache: judge CAS liveness on a coherent (word, clock) pair Eighth audit round (full re-read of cache.go, trie.go, and every test file; all six scenario classes re-traced), one Low: - casMatches sampled the clock before loading the expiry word (!l.expired(now())): now() was evaluated as the argument, then expired loaded the word. A CAS caller descheduled between the two judged a fresh word against a stale clock, so a kill that fully completed while it was parked went unseen: for Expire, the stored word n_e-1 compares above the stale clock; for a lapsing TTL, the fixed word never falls below a clock sampled before the lapse. Either way the publish lands against a value TryGet already certifies as Miss, and no sequential order explains kill, TryGet=Miss, CAS=true, TryGet=Hit(new) (a CAS-first order requires the final TryGet to miss the freshly swapped value; kill-first orders require the CAS to fail on the dead match). The sixth round closed the allocation-side window with the pre-publish re-validation, but the re-validation itself kept this instructions-wide clock-vs-word gap open on both kill modes. casMatches now takes its (word, clock) pair from expiresNow, exactly as every read path does: the clock postdates the word it judges, so any kill that completes before the pair is taken is always seen, and an idle extension racing the pair re-derives instead of spuriously failing a continuously-live match (the round-five misclassification class, which is why the simpler unconfirmed word-then-clock ordering was not an option). The residue is now exactly the instructions between the re-validation and the pointer CAS, which is what the CompareAndSwap / CompareAndDelete docs already document as best effort. Pinned by TestTryCAS_RevalidationPairsWordAndClock via a counting expiresPairingHook (parks the re-validation's paired read, the call's second), covering {CompareAndSwap, CompareAndDelete} x {TTL lapse, Expire}; all four subtests confirmed red against the pre-fix ordering with the hook held at the old deschedule point. The linearizability checker deliberately runs expiry-free, so the hook pin is the vehicle, as in rounds three through seven. Everything else re-verified clean this round: every other expiry-word read is either paired (entGet, TryGet), CAS-guarded (Expire, Clean's claim, idle extension), pinned to a deliberately conservative clock (Range, Delete, Swap's displaced-value report), or feeds a retry loop that re-derives before acting (Get's replace gate); the claim sentinel, tombstone terminality, walk single-visit, and prune lock ordering all re-traced clean. Full -race suite green; vet clean on native and linux/386. --- cache/cache.go | 24 +++++++--- cache/concurrency_test.go | 93 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 9 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index e58f8bf..f4f6d63 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -802,14 +802,24 @@ func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { // loading's v is whatever the miss function returned beside the error, // which was never cached as a value). // -// The liveness sample cannot be made atomic with the publishing CAS: a -// successful swap takes effect at the pointer CAS, and the clock cannot be -// read at that exact instant. tryCAS narrows the gap by re-validating -// immediately before each CAS attempt; the instructions-wide residue is -// documented on CompareAndSwap and CompareAndDelete. +// Liveness is judged on a coherent (expiry word, clock) pair from +// expiresNow, exactly as the read paths judge it: the clock postdates the +// word it is paired with, so a kill that completed before the pair was +// taken — a lapsed TTL, an Expire, a Clean claim — is always seen, no +// matter how long this call was descheduled mid-check, while an idle +// extension racing the pair re-derives rather than failing a +// continuously-live match. The pair still cannot be made atomic with the +// publishing CAS: a successful swap takes effect at the pointer CAS, and +// the clock cannot be read at that exact instant. tryCAS re-validates +// immediately before each CAS attempt; the instructions-wide residue +// between that check and the CAS is documented on CompareAndSwap and +// CompareAndDelete. func casMatches[V any](l *loading[V], old V) bool { - return l != nil && l.finalized() && l.err == nil && - any(l.v) == any(old) && !l.expired(now()) + if l == nil || !l.finalized() || l.err != nil || any(l.v) != any(old) { + return false + } + expires, n := l.expiresNow() + return expires == 0 || expires > n } /////////////////// diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index e193d26..ea80a06 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -1220,12 +1220,20 @@ func TestGet_CollapsedErrorWaitersRedrive(t *testing.T) { // first read to pass through it blocks until release is closed; all later // reads pass straight through. The hook is cleared when the test ends. func armExpiresPairingHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + return armExpiresPairingHookNth(t, 1) +} + +// armExpiresPairingHookNth is armExpiresPairingHook parking the nth +// (1-based) paired read instead of the first, for callers whose operation +// takes multiple paired reads before the one under test. +func armExpiresPairingHookNth(t *testing.T, nth int32) (entered, release chan struct{}) { t.Helper() entered = make(chan struct{}) release = make(chan struct{}) - var fired atomic.Bool + var calls atomic.Int32 expiresPairingHook = func() { - if fired.CompareAndSwap(false, true) { + if calls.Add(1) == nth { close(entered) <-release } @@ -1340,6 +1348,87 @@ func TestTryCAS_RevalidationCatchesExpire(t *testing.T) { } } +// TestTryCAS_RevalidationPairsWordAndClock pins HOW tryCAS's liveness +// checks read expiry: as a coherent (expiry word, clock) pair, the clock +// sampled after the word it judges (expiresNow, like every read path). A +// CAS caller descheduled inside the re-validation itself — between +// sampling the clock and loading the word, in the unpaired ordering — +// would otherwise judge the fresh word against its stale clock and +// publish against a value whose TTL had lapsed, or that a completed +// Expire had killed, while it was parked: state TryGet already certifies +// as Miss, so no sequential order explains TryGet=Miss followed by +// CAS=true followed by TryGet=Hit(new). With the paired read, any kill +// completed before the pair is taken is seen; only the instructions +// between the re-validation and the pointer CAS remain best effort (see +// CompareAndSwap's doc). +// +// The pairing hook parks the caller inside the re-validation's paired +// read — the call's second paired read; the first is tryCAS's +// pre-allocation filter — and the kill lands while it is parked. +// +// Regression test: casMatches used to evaluate now() and then load the +// word (!l.expired(now())), so a deschedule between the two let both kill +// modes through the re-validation. +func TestTryCAS_RevalidationPairsWordAndClock(t *testing.T) { + const ttl = 200 * time.Millisecond + ops := []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } + kills := []struct { + name string + opts []Opt + kill func(c *Cache[string, int]) + }{ + {"ttl_lapse", []Opt{MaxAge(ttl)}, func(c *Cache[string, int]) { + // Let the matched value's TTL lapse while the caller is + // parked; the certifying TryGets' own paired reads pass + // straight through the one-shot hook. + for { + if _, _, s := c.TryGet("k"); s.IsMiss() { + return + } + time.Sleep(time.Millisecond) + } + }}, + {"expire", []Opt{MaxAge(time.Hour)}, func(c *Cache[string, int]) { + c.Expire("k") + }}, + } + for _, k := range kills { + t.Run(k.name, func(t *testing.T) { + for _, m := range ops { + t.Run(m.name, func(t *testing.T) { + c := New[string, int](k.opts...) + c.Set("k", 1) + + entered, release := armExpiresPairingHookNth(t, 2) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + select { + case <-entered: + case r := <-res: + t.Fatalf("op returned %v before its re-validation (pre-filter failed; value dead too early?)", r) + } + + k.kill(c) + close(release) + + if <-res { + t.Fatal("published against a value that was killed mid-validation (TryGet had already certified Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + }) + } + }) + } +} + // TestTryGet_PairedReadVsIdleExtension pins the coherence of the (expiry // word, clock) pair a read classifies with: a reader descheduled between // loading the word and sampling the clock, with an idle extension landing From 87221d106cbb5b0ed89006b47bef7a1da551b488 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 22:18:38 -0600 Subject: [PATCH 17/21] cache: document Range's no-snapshot semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doc accuracy, from the eighth audit round: Cache.Range promised "calls fn for every cached value" with no concurrency caveat. The trie walk takes no snapshot — the old read/dirty implementation ranged a promoted read map, which did fix the key set at the start of the call, so the port silently weakened Range's consistency while the doc stayed one line. State the actual contract, in sync.Map.Range's terms: no key is visited more than once (pinned by TestConcurrentRange's seen-twice check under churn), a concurrently mutated or expiring key may be reflected from any point during the call (Range classifies every entry at one clock pinned at entry), and Range does not block other methods — fn may re-enter the cache (pinned by TestMapRangeNestedCall). Set.Range points at the Cache caveat, following the Item "See Cache.Get" precedent. --- cache/cache.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cache/cache.go b/cache/cache.go index f4f6d63..a14198e 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -516,6 +516,12 @@ func (c *Cache[K, V]) Expire(k K) { } // Range calls fn for every cached value. If fn returns false, iteration stops. +// +// Range does not necessarily correspond to any consistent snapshot of the +// cache: no key is visited more than once, but if a key is set, deleted, or +// expires concurrently (including from fn itself), Range may reflect any +// state of that key from any point during the call. Range does not block +// other methods on the cache; even fn may call them. func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // When ranging, repeated time.Now() calls add up, so we get the // current time when we enter range and avoid it in all tryGet calls. @@ -1303,6 +1309,8 @@ func (s *Set[K]) Expire(k K) { } // Range calls fn for every cached key. If fn returns false, iteration stops. +// See Cache.Range for the consistency caveats: no snapshot is taken, and no +// key is visited more than once. func (s *Set[K]) Range(fn func(K, error) bool) { s.c.Range(func(k K, _ struct{}, err error) bool { return fn(k, err) From 052940d17bcedb37b29068819baefabecf153c25 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 22:49:14 -0600 Subject: [PATCH 18/21] cache: document Get's expired-with-stale return MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doc accuracy, from the ninth audit round: the Get family's stale sentence covered only the errored case — 'if stale values are enabled, the currently cached value has an error, and there is an unexpired stale value, this returns the stale value and no error' — wording inherited verbatim from the read/dirty implementation. The third round taught TryGet's doc that an expired value with an open stale window is returned as Stale, and the fourth round made that window a first-class read state (derived from the expiry word, agreed on by every read and refresh), but Get's doc never learned it: a Get on an expired value with an open window returns the value as Stale immediately while driving the refresh, and it re-runs the miss function for an expired key in the first place — neither of which the doc stated. KeyState's Stale doc and MaxStaleAge already describe both kill modes, so Get was the one surface left claiming the narrower contract. Get, Item.Get, and Set.Get now say it the way TryGet does: the miss runs if the key is not yet cached or the cached value has expired, and an unexpired stale — the value errored, or expired with its window open — is returned with no error. Behavior unchanged and already pinned: TestExpires asserts Get returns (v, nil, Stale) for an expired-with-window value, and TestGet_ConcurrentStaleDuringInFlight pins the piggybacked flavor. --- cache/cache.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index a14198e..a28d932 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -316,9 +316,10 @@ func initCache[K comparable, V any](c *Cache[K, V], opts ...Opt) { } // Get returns the cache value for k, running the miss function in a goroutine -// if the key is not yet cached. If stale values are enabled, the currently -// cached value has an error, and there is an unexpired stale value, this -// returns the stale value and no error. +// if the key is not yet cached or the cached value has expired. If stale +// values are enabled and an unexpired stale value exists — the currently +// cached value has an error, or it expired with its stale window still open — +// this returns the stale value and no error. // // The miss function runs on an internal goroutine: if it panics, the process // crashes (the panic cannot be recovered by the Get caller); recover inside @@ -1156,10 +1157,11 @@ func NewItem[V any](opts ...Opt) *Item[V] { } // Get returns the currently cached value, running the miss function in a -// goroutine if the item is not yet cached. If stale values are enabled, the -// currently cached value has an error, and there is an unexpired stale value, -// this returns the stale value and no error. See Cache.Get for the miss -// function's panic and re-entrancy caveats. +// goroutine if the item is not yet cached or the cached value has expired. If +// stale values are enabled and an unexpired stale value exists — the +// currently cached value has an error, or it expired with its stale window +// still open — this returns the stale value and no error. See Cache.Get for +// the miss function's panic and re-entrancy caveats. func (i *Item[V]) Get(miss func() (V, error)) (v V, err error, state KeyState) { return i.c.Get(struct{}{}, miss) } @@ -1268,10 +1270,11 @@ func NewSet[K comparable](opts ...Opt) *Set[K] { } // Get ensures the key is cached, running the miss function in a goroutine if -// the key is not yet cached. If stale keys are enabled, the currently cached -// key has an error, and there is a stale key, this returns with no error and a -// Stale key state. See Cache.Get for the miss function's panic and -// re-entrancy caveats. +// the key is not yet cached or the cached key has expired. If stale keys are +// enabled and an unexpired stale exists — the currently cached key has an +// error, or it expired with its stale window still open — this returns with +// no error and a Stale key state. See Cache.Get for the miss function's +// panic and re-entrancy caveats. func (s *Set[K]) Get(k K, miss func() error) (err error, state KeyState) { _, err, state = s.c.Get(k, func() (struct{}, error) { return struct{}{}, miss() From 49c570726adf6928e65aa2a5d0039348a5dbe1a8 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 22:50:01 -0600 Subject: [PATCH 19/21] cache: MaxErrorAge's default accounts for MaxIdleAge's initial TTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doc accuracy, from the ninth audit round: MaxErrorAge claimed 'with neither option set, a load error is cached forever and never retried'. The fourth round wrote that sentence weighing only MaxAge; it never weighed MaxIdleAge, whose initial-TTL rule in newExpires applies to every entry — the err branch falls through to the same ttl derivation, so with only MaxIdleAge set a load error is cached for the idle age and then retried, not cached forever (verified live: error Hit within the window, Miss after, next Get re-drives). The behavior is the sensible one — an idle-config cache should not pin errors permanently — so the doc moves to the code: the default is MaxAge, or the idle age when only MaxIdleAge is set (errors share the initial TTL but are never idle-extended; extension is gated on err == nil everywhere). The forever claim is rescoped to neither-MaxAge-nor-MaxIdleAge. Pinned by TestMaxErrorAge_DefaultsToIdleInitialTTL: cached error is a Hit within the idle window, Miss after it lapses, and the next Get drives a fresh load. The no-extension half was already pinned for the MaxAge config by TestMaxIdleAge/no_extend_errors; the new test's early probe runs immediately after the load so even a regressed extension could not outlive the post-window probe. --- cache/cache.go | 9 ++++++--- cache/cache_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index a28d932..b72dda4 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -225,9 +225,12 @@ func MaxAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxAge, c.a func MaxStaleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxStaleAge = age }} } // MaxErrorAge sets the age to persist load errors. If not specified, the -// default is MaxAge — so with neither option set, a load error is cached -// forever and never retried (until Set, Swap, Delete, or Expire). Using -// this option with 0 disables caching errors entirely. +// default is MaxAge — or the idle age, when only MaxIdleAge is set, since +// the idle age is then every entry's initial TTL (errors included, though an +// errored entry is never idle-extended). With neither MaxAge nor MaxIdleAge +// set, a load error is cached forever and never retried (until Set, Swap, +// Delete, or Expire). Using this option with 0 disables caching errors +// entirely. // // A cached error suppresses repeat queries only when there is no stale // value to return: if stale values are enabled and a valid stale exists, diff --git a/cache/cache_test.go b/cache/cache_test.go index 6aabc65..c573c22 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -461,6 +461,36 @@ func TestMaxIdleAge(t *testing.T) { }) } +// TestMaxErrorAge_DefaultsToIdleInitialTTL pins the error-TTL default when +// only MaxIdleAge is set (see MaxErrorAge's doc): the idle age is every +// entry's initial TTL, errors included, so a load error is cached for the +// idle age — not forever — and the next Get after it lapses retries. The +// pre-expiry probe asserts the error is the cached result within the +// window; it cannot mask a (regressed) idle extension of the errored entry, +// because it runs immediately after the load, so even an extension anchored +// at that access would lapse before the post-window probe. +func TestMaxErrorAge_DefaultsToIdleInitialTTL(t *testing.T) { + const idle = 200 * time.Millisecond + c := New[string, int](MaxIdleAge(idle)) + boom := errors.New("boom") + c.Get("k", func() (int, error) { return 0, boom }) + + if _, err, s := c.TryGet("k"); err == nil || !s.IsHit() { + t.Fatalf("TryGet within the idle window: err=%v s=%v, want the cached error as a Hit", err, s) + } + + time.Sleep(2*idle + 50*time.Millisecond) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the idle age: %v, want Miss (the error is not cached forever)", s) + } + + var retried bool + v, err, s := c.Get("k", func() (int, error) { retried = true; return 7, nil }) + if !retried || v != 7 || err != nil || s != Miss { + t.Fatalf("Get after the error lapsed: retried=%v (%d, %v, %v), want a fresh (7, nil, Miss) load", retried, v, err, s) + } +} + // TestHugeAges verifies expiry arithmetic saturates rather than wrapping: // absurd-but-valid Durations must mean "effectively forever", not "born // expired" (or, for Clean, "evict everything"). From 362321af10117c05d142de78737f9b250c2c3048 Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 23:38:19 -0600 Subject: [PATCH 20/21] cache: Get's miss-run conditions include the errored-with-stale refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doc accuracy, from the tenth audit round: Get's first sentence claimed the miss function runs "if the key is not yet cached or the cached value has expired" — an exhaustive-reading condition that misses the third trigger. A cached error whose own age has not lapsed is replaced and re-driven whenever a valid stale exists: the slow path's replacement filter passes any finalized errored loading regardless of expiry, exactly as MaxErrorAge documents ("a cached error suppresses repeat queries only when there is no stale value to return ... always drives a refresh, regardless of the error's remaining age"). The ninth round's rewrite (052940d) brought the expired-with-stale return into Get's doc but enumerated only the two run triggers, leaving Get contradicting MaxErrorAge for live errors. Get, Item.Get, and Set.Get now list the third trigger, with the MaxErrorAge rationale parenthesized on Cache.Get. Behavior unchanged and already pinned: TestExpires's stale-while-error block returns Stale from a Get whose miss function is concurrently executing (blocked on a channel) while the cached error is unexpired (no error age is configured there, so the error would otherwise be cached and served forever). --- cache/cache.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index b72dda4..c8598a5 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -319,8 +319,10 @@ func initCache[K comparable, V any](c *Cache[K, V], opts ...Opt) { } // Get returns the cache value for k, running the miss function in a goroutine -// if the key is not yet cached or the cached value has expired. If stale -// values are enabled and an unexpired stale value exists — the currently +// if the key is not yet cached, the cached value has expired, or the cached +// value is an unexpired error with a valid stale to serve (a cached error +// suppresses repeat queries only while no stale exists; see MaxErrorAge). If +// stale values are enabled and an unexpired stale value exists — the currently // cached value has an error, or it expired with its stale window still open — // this returns the stale value and no error. // @@ -1160,7 +1162,8 @@ func NewItem[V any](opts ...Opt) *Item[V] { } // Get returns the currently cached value, running the miss function in a -// goroutine if the item is not yet cached or the cached value has expired. If +// goroutine if the item is not yet cached, the cached value has expired, or +// the cached value is an unexpired error with a valid stale to serve. If // stale values are enabled and an unexpired stale value exists — the // currently cached value has an error, or it expired with its stale window // still open — this returns the stale value and no error. See Cache.Get for @@ -1273,7 +1276,8 @@ func NewSet[K comparable](opts ...Opt) *Set[K] { } // Get ensures the key is cached, running the miss function in a goroutine if -// the key is not yet cached or the cached key has expired. If stale keys are +// the key is not yet cached, the cached key has expired, or the cached key +// holds an unexpired error with a valid stale to serve. If stale keys are // enabled and an unexpired stale exists — the currently cached key has an // error, or it expired with its stale window still open — this returns with // no error and a Stale key state. See Cache.Get for the miss function's From 4b221c4cf11e4de0afc7dd7d7696f9a9a249a2bf Mon Sep 17 00:00:00 2001 From: Travis Bischel Date: Thu, 11 Jun 2026 23:40:04 -0600 Subject: [PATCH 21/21] cache: document Clear's in-flight-load semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doc accuracy, from the tenth audit round: Clear's doc said only "deletes all keys, resetting it to an empty state", leaving its interplay with in-flight loads underivable. Clear swaps the trie root, so a load in flight at the Clear keeps running against the detached tree: its result still reaches every Get waiting on it (the waiters hold the detached entry and its loading directly), but it is never cached — a later Get for the same key misses and drives a fresh, independent load. This is the per-key Delete caveat applied wholesale, but nothing said Clear behaves like a bulk Delete rather than, say, canceling loads or letting their results re-land in the emptied cache. Cache.Clear now states it; Item.Clear and Set.Clear point at it, following the Item.CompareAndSwap "see Cache.X" precedent. Pinned by TestClear_DoesNotCacheInFlightLoads, the Clear sibling of TestDelete_DuringInFlightLoad: with the miss held open across the Clear, the waiter still receives the load's value, and the post-Clear Get re-runs the miss instead of finding the detached result. --- cache/cache.go | 11 +++++++--- cache/concurrency_test.go | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index c8598a5..1c247aa 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -615,7 +615,10 @@ func (c *Cache[K, V]) Clean() { } } -// Clear deletes all keys from the cache, resetting it to an empty state. +// Clear deletes all keys from the cache, resetting it to an empty state. A +// load in flight at the time of the Clear still completes and returns its +// result to the Gets waiting on it, but the result is not cached: a later +// Get for the same key misses and drives a fresh, independent load. func (c *Cache[K, V]) Clear() { c.t.clear() } @@ -1232,7 +1235,8 @@ func (i *Item[V]) CompareAndDelete(old V) (deleted bool) { return i.c.CompareAndDelete(struct{}{}, old) } -// Clear deletes the cached item, resetting the item to an empty state. +// Clear deletes the cached item, resetting the item to an empty state. See +// Cache.Clear for the in-flight-load caveat. func (i *Item[V]) Clear() { i.c.Clear() } @@ -1346,7 +1350,8 @@ func (s *Set[K]) Set(k K) { s.c.Set(k, struct{}{}) } -// Clear deletes all keys from the set, resetting it to an empty state. +// Clear deletes all keys from the set, resetting it to an empty state. See +// Cache.Clear for the in-flight-load caveat. func (s *Set[K]) Clear() { s.c.Clear() } diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go index ea80a06..e482056 100644 --- a/cache/concurrency_test.go +++ b/cache/concurrency_test.go @@ -143,6 +143,52 @@ func TestDelete_DuringInFlightLoad(t *testing.T) { } } +// TestClear_DoesNotCacheInFlightLoads pins Clear's documented interplay with +// in-flight loads (the per-key Delete caveat applied wholesale): Clear swaps +// the trie root, so a load in flight at the Clear keeps running against the +// detached tree — its result still reaches the Get waiting on it, but is +// never cached, and a later Get misses and drives a fresh load. +func TestClear_DoesNotCacheInFlightLoads(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Add(1) + go func() { + defer getDone.Done() + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 5, nil + }) + getV = v + }() + + <-missStart + c.Clear() + close(missRelease) + getDone.Wait() + + if getV != 5 { + t.Fatalf("in-flight Get got %d, want 5 (the load's result must still reach its waiter)", getV) + } + + // The result must not have been cached: the next Get re-runs the miss. + var called bool + v, _, _ := c.Get("k", func() (int, error) { + called = true + return 6, nil + }) + if !called { + t.Fatal("miss function should have run after Clear (the in-flight result must not be cached)") + } + if v != 6 { + t.Fatalf("post-Clear Get got %d, want 6", v) + } +} + // TestExpire_NoOpDuringInFlightLoad verifies the documented behavior: Expire // on an in-flight load is a no-op; the load completes with its normal TTL. func TestExpire_NoOpDuringInFlightLoad(t *testing.T) {