diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0766e66 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: ci + +on: + push: + branches: ["main"] + pull_request: + +jobs: + test: + # amd64 and arm64: the trie/cache core is lock-free, and arm64's weaker + # memory ordering surfaces reordering bugs that x86's TSO hides. + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: stable + - run: go vet ./... + - run: go test -race ./... + + test-386: + # 32-bit: uintptr hashes are 32 bits, so the trie is 8 levels deep + # instead of 16 and the test hash constants must fit the word size. + # The race detector is unsupported on 386, so this runs without it. + name: test (linux/386) + runs-on: ubuntu-latest + env: + GOARCH: "386" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: stable + - run: go vet ./... + - run: go test ./... diff --git a/README.md b/README.md index 64bb839..57d0858 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,13 @@ cache Package cache provides a concurrency safe, mostly lock-free, singleflight request collapsing generic cache with support for stale values. -The guts of this package are similar to `sync.Map`, with minor differences in -when the internal locked write-map is promoted an atomic read-only map, and -major differences to support singleflight request collapsing. +The internal storage is a concurrent hash-trie, modeled on Go's +`internal/sync.HashTrieMap` (the backing of the current `sync.Map`) but +implemented entirely in userspace via `hash/maphash` for the runtime's typed +hasher. Loads are lock-free; inserts and deletes take only a per-bucket +mutex, never a global one. On top of that storage the cache layers +singleflight request collapsing, stale values, TTLs, idle-age extension, and +error caching. Functions in this API that return a `KeyState` return the state last, rather than the error last: the value and error are cached as a single internal unit @@ -34,9 +38,22 @@ value can be kept and returned from `Get` during a refresh / if a refresh fails. Keys can be manually expired with `Expire`. Internally expired values or errors can be occasionally cleaned with `Clean`. -Out of an abundance of paranoia that this code is correct, there are unit tests -to hit 97% coverage of the `cache.Cache` type. As well, all tests against -`sync.Map` are copied into this library and used against `cache.Cache`. +Out of an abundance of paranoia that this code is correct, the test suite +layers several kinds of verification: unit tests at ~99% statement coverage +of the cache and trie (the remainder being defensive panics at +invariant-violation points and race-retry branches that require +interleavings that cannot be forced from userspace); all of the stdlib +`sync.Map` tests, copied into this library and run against `cache.Cache`; a +property test comparing the cache to a `sync.RWMutex`-guarded `map` oracle +over random operation sequences; a linearizability checker that runs batches +of concurrent operations against single keys and verifies that a +real-time-consistent sequential ordering explains every observed return; +structural validation of the trie (hash placement, overflow chains, parent +pointers, pruning) after each concurrent stress test; and regression hammers +for specific interleaving bugs found by audit. CI runs the suite under the +race detector on both amd64 and arm64 (whose weaker memory ordering +surfaces reordering bugs that x86 hides), and without it on 32-bit +linux/386. This package also provides a cached `Item` and a `Set`. `Item` can be used to populate an expensive value once and expire or replace it when needed, similar @@ -54,87 +71,40 @@ Documentation Benchmarking ------------ -Some of the APIs are actually faster or more memory efficient than `sync.Map` -because of generics. Some are slower and require more allocs because of the -request collapsing aspect. Some benchmarks below show more CPU, but this is due -to a few slight changes in how the write (dirty) map is internally promoted to -the read map. - -The benchmarks are from the stdlib using a key and value of type int: +Microbenchmarks on an Apple M1, Go 1.26, against the `CacheMap[int]` wrapper +(`cache.Cache[int, any]` adapted to the `sync.Map` `mapInterface` so the +stdlib benchmarks apply directly): ``` -name old time/op new time/op delta -LoadMostlyHits 12.8ns ±44% 14.0ns ±50% ~ (p=0.393 n=10+10) -LoadMostlyMisses 10.7ns ±53% 4.6ns ±47% -57.31% (p=0.000 n=10+10) -LoadOrStoreBalanced 351ns ± 3% 705ns ±27% +100.68% (p=0.000 n=8+10) -LoadOrStoreUnique 663ns ±13% 1296ns ±31% +95.53% (p=0.000 n=10+10) -LoadOrStoreCollision 9.71ns ±60% 29.26ns ±46% +201.44% (p=0.000 n=10+10) -LoadAndDeleteBalanced 11.2ns ±42% 7.1ns ±48% -36.06% (p=0.035 n=10+10) -LoadAndDeleteUnique 9.19ns ±62% 4.17ns ±47% -54.59% (p=0.001 n=10+10) -LoadAndDeleteCollision 10.9ns ±41% 14.3ns ±35% +31.18% (p=0.023 n=10+10) -Range 3.73µs ±48% 5.40µs ±45% ~ (p=0.063 n=10+10) -AdversarialAlloc 298ns ±22% 259ns ±21% ~ (p=0.063 n=10+10) -AdversarialDelete 83.7ns ±14% 105.6ns ±20% ~ (p=0.052 n=10+10) -DeleteCollision 6.87ns ±71% 4.86ns ±100% ~ (p=0.424 n=10+10) -SwapCollision 201ns ±26% 266ns ±31% ~ (p=0.143 n=10+10) -SwapMostlyHits 40.3ns ±42% 59.7ns ±51% ~ (p=0.143 n=10+10) -SwapMostlyMisses 662ns ±18% 607ns ±24% -8.19% (p=0.023 n=10+10) -CompareAndSwapCollision 33.4ns ±41% 24.9ns ±65% -25.44% (p=0.023 n=10+10) -CompareAndSwapNoExistingKey 10.3ns ±46% 2.0ns ±53% -80.15% (p=0.000 n=10+10) -CompareAndSwapValueNotEqual 9.64ns ±51% 6.02ns ±95% -37.49% (p=0.019 n=10+10) -CompareAndSwapMostlyHits 46.9ns ±39% 56.8ns ±47% ~ (p=0.143 n=10+10) -CompareAndSwapMostlyMisses 22.9ns ±39% 13.4ns ±48% -41.48% (p=0.023 n=10+10) -CompareAndDeleteCollision 28.1ns ±47% 18.3ns ±37% -34.88% (p=0.019 n=10+10) -CompareAndDeleteMostlyHits 64.0ns ±38% 63.9ns ±52% ~ (p=0.165 n=10+10) -CompareAndDeleteMostlyMisses 18.3ns ±37% 8.7ns ±53% -52.65% (p=0.023 n=10+10) - -name old alloc/op new alloc/op delta -LoadMostlyHits 6.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadMostlyMisses 6.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadOrStoreBalanced 74.7B ±16% 195.3B ± 1% +161.45% (p=0.000 n=10+10) -LoadOrStoreUnique 159B ± 7% 330B ± 7% +107.15% (p=0.000 n=9+10) -LoadOrStoreCollision 0.00B 32.00B ± 0% +Inf% (p=0.000 n=10+10) -LoadAndDeleteBalanced 4.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadAndDeleteUnique 8.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -LoadAndDeleteCollision 1.00B ± 0% 1.00B ± 0% ~ (all equal) -Range 16.0B ± 0% 0.0B -100.00% (p=0.000 n=10+10) -AdversarialAlloc 53.4B ± 3% 64.8B ± 3% +21.35% (p=0.000 n=10+10) -AdversarialDelete 22.6B ±12% 39.0B ± 0% +72.57% (p=0.000 n=10+10) -DeleteCollision 0.00B 0.00B ~ (all equal) -SwapCollision 16.0B ± 0% 80.0B ± 0% +400.00% (p=0.000 n=10+10) -SwapMostlyHits 28.0B ± 0% 86.0B ± 0% +207.14% (p=0.000 n=10+10) -SwapMostlyMisses 124B ± 0% 136B ± 1% +9.71% (p=0.000 n=10+10) -CompareAndSwapCollision 7.60B ± 8% 21.70B ±17% +185.53% (p=0.000 n=10+10) -CompareAndSwapNoExistingKey 8.00B ± 0% 0.00B -100.00% (p=0.000 n=10+10) -CompareAndSwapValueNotEqual 0.00B 0.00B ~ (all equal) -CompareAndSwapMostlyHits 33.0B ± 0% 91.0B ± 0% +175.76% (p=0.000 n=10+10) -CompareAndSwapMostlyMisses 23.0B ± 0% 16.0B ± 0% -30.43% (p=0.000 n=10+10) -CompareAndDeleteCollision 0.00B 2.00B ± 0% +Inf% (p=0.000 n=10+8) -CompareAndDeleteMostlyHits 39.0B ± 0% 90.6B ± 1% +132.31% (p=0.000 n=10+10) -CompareAndDeleteMostlyMisses 16.0B ± 0% 8.0B ± 0% -50.00% (p=0.002 n=8+10) - -name old allocs/op new allocs/op delta -LoadMostlyHits 0.00 0.00 ~ (all equal) -LoadMostlyMisses 0.00 0.00 ~ (all equal) -LoadOrStoreBalanced 2.00 ± 0% 3.00 ± 0% +50.00% (p=0.000 n=10+10) -LoadOrStoreUnique 4.00 ± 0% 5.00 ± 0% +25.00% (p=0.000 n=10+10) -LoadOrStoreCollision 0.00 1.00 ± 0% +Inf% (p=0.000 n=10+10) -LoadAndDeleteBalanced 0.00 0.00 ~ (all equal) -LoadAndDeleteUnique 0.60 ±100% 0.00 -100.00% (p=0.011 n=10+10) -LoadAndDeleteCollision 0.00 0.00 ~ (all equal) -Range 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -AdversarialAlloc 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -AdversarialDelete 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) -DeleteCollision 0.00 0.00 ~ (all equal) -SwapCollision 1.00 ± 0% 1.00 ± 0% ~ (all equal) -SwapMostlyHits 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) -SwapMostlyMisses 6.00 ± 0% 4.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndSwapCollision 0.00 0.00 ~ (all equal) -CompareAndSwapNoExistingKey 0.50 ±100% 0.00 -100.00% (p=0.033 n=10+10) -CompareAndSwapValueNotEqual 0.00 0.00 ~ (all equal) -CompareAndSwapMostlyHits 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndSwapMostlyMisses 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) -CompareAndDeleteCollision 0.00 0.00 ~ (all equal) -CompareAndDeleteMostlyHits 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.000 n=10+10) -CompareAndDeleteMostlyMisses 1.00 ± 0% 0.00 -100.00% (p=0.000 n=10+10) +name time/op alloc/op allocs/op +LoadMostlyHits 10.3 ns/op 0 B/op 0 allocs/op +LoadMostlyMisses 5.2 ns/op 0 B/op 0 allocs/op +LoadOrStoreBalanced 448 ns/op 151 B/op 3 allocs/op +LoadOrStoreUnique 567 ns/op 262 B/op 5 allocs/op +LoadOrStoreCollision 45.6 ns/op 32 B/op 1 allocs/op +LoadAndDeleteBalanced 4.88 ns/op 0 B/op 0 allocs/op +LoadAndDeleteUnique 2.00 ns/op 0 B/op 0 allocs/op +LoadAndDeleteCollision 20.2 ns/op 2 B/op 0 allocs/op +Range 8.64 µs/op 0 B/op 0 allocs/op +AdversarialAlloc 11.7 ns/op 0 B/op 0 allocs/op +AdversarialDelete 7.17 ns/op 0 B/op 0 allocs/op +DeleteCollision 3.69 ns/op 0 B/op 0 allocs/op +SwapCollision 243 ns/op 80 B/op 1 allocs/op +SwapMostlyHits 119 ns/op 86 B/op 1 allocs/op +SwapMostlyMisses 160 ns/op 120 B/op 2 allocs/op +CompareAndSwapCollision 13.1 ns/op 3 B/op 0 allocs/op +CompareAndSwapNoExistingKey 2.57 ns/op 0 B/op 0 allocs/op +CompareAndSwapValueNotEqual 10.6 ns/op 0 B/op 0 allocs/op +CompareAndSwapMostlyHits 128 ns/op 91 B/op 2 allocs/op +CompareAndSwapMostlyMisses 30.1 ns/op 16 B/op 1 allocs/op +CompareAndDeleteCollision 13.0 ns/op 0 B/op 0 allocs/op +CompareAndDeleteMostlyHits 112 ns/op 91 B/op 2 allocs/op +CompareAndDeleteMostlyMisses 13.3 ns/op 8 B/op 0 allocs/op ``` + +The per-op allocations on the `LoadOrStore*`, `Swap*`, and `CompareAndSwap*Hits` +rows are intrinsic: each of these operations may install a fresh `loading[V]` +in the trie's value slot, and that struct carries the singleflight machinery +(`sync.Mutex`, `sync.WaitGroup`, atomic counters) that a plain `sync.Map` does +not need. The lock-free-read paths (`Load*`, `Delete*`, `Range`, the +`Adversarial*` patterns, and the `CompareAnd*Misses` paths) are alloc-free. diff --git a/cache/autoclean_test.go b/cache/autoclean_test.go new file mode 100644 index 0000000..ab2e5d1 --- /dev/null +++ b/cache/autoclean_test.go @@ -0,0 +1,116 @@ +package cache + +import ( + "sync/atomic" + "testing" + "time" +) + +func TestAutoClean_RunsAndCleansExpired(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(2*time.Millisecond), + ) + defer c.StopAutoClean() + + c.Set("a", 1) + if _, _, s := c.TryGet("a"); !s.IsHit() { + t.Fatalf("immediately after Set: want Hit") + } + + // Wait long enough for the ticker to tick at least a few times. + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if _, _, s := c.TryGet("a"); s.IsMiss() { + return // AutoClean fired and removed the expired entry. + } + time.Sleep(2 * time.Millisecond) + } + t.Fatal("AutoClean never removed the expired entry") +} + +func TestAutoClean_DisabledWhenStaleAgeNegative(t *testing.T) { + // AutoCleanInterval is ignored when MaxStaleAge < 0 (stales kept forever). + // We verify no goroutine is spawned by confirming StopAutoClean is a no-op + // on a cache whose quitClean channel was never created. + c := New[string, int]( + MaxAge(time.Millisecond), + MaxStaleAge(-1), + AutoCleanInterval(time.Millisecond), + ) + if c.quitClean != nil { + t.Fatal("quitClean channel should not be allocated when MaxStaleAge<0") + } + c.StopAutoClean() // no-op must not panic +} + +func TestAutoClean_StopIdempotent(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(time.Millisecond), + ) + c.StopAutoClean() + c.StopAutoClean() // sync.Once guards the close; second call must be safe. +} + +func TestAutoClean_StopOnCacheWithoutGoroutine(t *testing.T) { + // A cache created without AutoCleanInterval has no goroutine and no + // quitClean channel. StopAutoClean must still be safe to call. + c := New[string, int]() + c.StopAutoClean() +} + +func TestAutoClean_ZeroIntervalNoGoroutine(t *testing.T) { + // autoCleanInterval == 0 means disabled; no goroutine, no channel. + c := New[string, int](MaxAge(time.Second)) + if c.quitClean != nil { + t.Fatal("quitClean channel should not be allocated with zero interval") + } + c.StopAutoClean() +} + +// Verifies that once StopAutoClean is called, the goroutine actually exits and +// subsequent ticks are not observed. We can't directly observe goroutine exit +// but we can observe Clean not running: after stopping, a freshly expired +// entry must remain until the test manually cleans. +func TestAutoClean_StopHaltsCleaning(t *testing.T) { + var cleanCount int32 + c := New[string, int]( + MaxAge(time.Millisecond), + AutoCleanInterval(time.Millisecond), + ) + // Poll until we see at least one clean happened by observing entry removal. + c.Set("a", 1) + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if _, _, s := c.TryGet("a"); s.IsMiss() { + atomic.StoreInt32(&cleanCount, 1) + break + } + time.Sleep(time.Millisecond) + } + if atomic.LoadInt32(&cleanCount) == 0 { + t.Fatal("AutoClean never observed cleaning before stop") + } + + c.StopAutoClean() + + // After stop, set an entry and verify it is NOT auto-cleaned within a + // window many multiples of the original interval. + c.Set("b", 2) + time.Sleep(20 * time.Millisecond) + // b is expired (MaxAge=1ms) but AutoClean is stopped, so TryGet still + // reports Miss only because of the expiry check — the key remains in + // the map. Verify Clean manually removes it. + c.Clean() + // A subsequent Get with a non-panicking miss should be called (i.e., the + // key is actually gone from the map, not just expired). + var missCalled bool + c.Get("b", func() (int, error) { + missCalled = true + return 0, nil + }) + if !missCalled { + t.Fatal("miss function should have run after manual Clean removed the entry") + } +} diff --git a/cache/cache.go b/cache/cache.go index 0e6d5af..1c247aa 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -1,9 +1,10 @@ // Package cache provides a concurrency safe, mostly lock-free, singleflight // request collapsing generic cache with support for stale values. // -// The guts of this package are similar to `sync.Map`, with minor differences -// in when the internal locked write-map is promoted an atomic read-only map, -// and major differences to support singleflight request collapsing. +// The internal storage is a concurrent hash-trie (see trie.go), inspired by +// Go's internal/sync.HashTrieMap. Loads are lock-free; inserts and deletes +// take a per-bucket (not global) mutex. The cache layers singleflight, +// stale values, TTLs, idle-age extension, and error caching on top. // // Functions in this API that return a KeyState return the state last, rather // than the error last: the value and error are cached as a single internal @@ -16,6 +17,7 @@ package cache import ( + "math" "sync" "sync/atomic" "time" @@ -49,37 +51,41 @@ type ( expires int64 // nano at which this stale ent is unusable, if non-zero } loading[V any] struct { - v V - err error - expires atomic.Int64 // nano at which this ent is unusable, if non-zero - stale *stale[V] - state atomic.Uint32 + v V + err error + // expires is the entry's expiry word; see the grammar documented + // at expiredClaimed below. + expires atomic.Int64 + // stale is the refresh companion: the previous generation's value, + // snapshotted when this loading was installed to replace an + // expired or errored predecessor (entMaybeNewStale), and served + // while this loading is in flight or while its result is an + // error. It is immutable once the loading is published. An + // expired err-free value's own post-expiry stale window is not + // stored anywhere; it is derived from the expires word at read + // time (see staleOpen). + stale *stale[V] + state atomic.Uint32 wg sync.WaitGroup mu sync.Mutex } - ent[V any] struct { - p atomic.Pointer[loading[V]] - } - read[K comparable, V any] struct { - m map[K]*ent[V] - incomplete bool - } // Cache caches comparable keys to arbitrary values. By default, the // cache grows without bounds and all keys persist forever. These // limits can be changed with options that are passed to New. + // + // Keys behave like the builtin map's: a key containing a NaN is + // never equal to itself, so it can be stored but never found, + // deleted, or cleaned again — every Get with such a key runs the + // miss function and adds an entry that only Clear removes. Likewise, + // an interface key holding a non-comparable dynamic value panics, + // as with a map. Cache[K comparable, V any] struct { - r atomic.Pointer[read[K, V]] - mu sync.Mutex - dirty map[K]*ent[V] - - misses int + t trie[K, loading[V]] cfg cfg - pd *loading[V] // allocate the promotingDelete pointer once - quitOnce sync.Once quitClean chan struct{} } @@ -105,10 +111,58 @@ type ( const ( stateLoading uint32 = iota stateFinalized - statePromotingDelete ) -func now() int64 { return time.Now().UnixNano() } +// The grammar of the loading.expires word. Positive values are +// monotonic-clock nanos (see epoch); all negative values are expired at +// every clock sample, since now is always strictly positive: +// +// 0: load in flight, or — once finalized — "never expires" +// > 0: expires at that nano +// -2: expiredClaimed, Clean's claim: the value is expired and its +// stale window is certified closed +// < -2: born expired (MaxAge(0) / MaxErrorAge(0)), carrying the negated +// birth nano so the value's stale window can anchor at the birth +// (see newExpires, staleOpen, newStale) +// +// The epoch's one-second pad keeps now() at or above a second of nanos, so +// a negated birth can never collide with expiredClaimed. The two negative +// shapes are deliberately distinct: a born-expired value's stale window is +// derived from its birth, while a claimed word means the window was +// certified closed and must never be re-derived (see staleOpen and +// entMaybeNewStale). +const expiredClaimed = -2 + +// ent is the concrete type of cache entries living in the trie. Using a +// type alias (Go 1.24+) lets us refer to entries by a short name while the +// trie stays a generic hash-trie with no cache-specific logic. +type ent[K comparable, V any] = trieEntry[K, loading[V]] + +// epoch anchors now. Expiry math uses the monotonic clock so that two +// causally ordered samples can never run backwards: entGet's "load the +// expiry, then sample the clock" argument — and Expire reliably winning +// races against idle extension — relies on that, and a wall-clock step +// (NTP, manual adjustment) would otherwise move every TTL. The one-second +// pad keeps now strictly positive so it can never collide with the 0 +// sentinels ("no expiry" / "caller samples the clock"), and keeps negated +// births (see the expires-word grammar) below expiredClaimed. Trade-off: on +// platforms whose monotonic clock pauses during system suspend, entries +// age only while the system runs. +var epoch = time.Now().Add(-time.Second) + +func now() int64 { return int64(time.Since(epoch)) } + +// satAdd returns a+b saturated at math.MaxInt64, for expiry arithmetic on +// nanosecond quantities; b must be non-negative, and callers pass a > 0 +// (sentinel expiries are filtered before the math). Without saturation, a +// huge MaxAge / MaxStaleAge / MaxIdleAge would wrap negative and make +// entries born (or instantly) expired. +func satAdd(a, b int64) int64 { + if s := a + b; s >= a { + return s + } + return math.MaxInt64 +} func (cfg *cfg) newExpires(err error) int64 { var ttl time.Duration @@ -124,12 +178,17 @@ func (cfg *cfg) newExpires(err error) int64 { del0 = cfg.ageSet && cfg.maxAge <= 0 } if del0 { - return -1 + // Born expired: negative, so the value is expired at every clock + // sample, carrying the negated birth so its stale window anchors + // at the birth rather than at whatever read happens to derive it + // (see staleOpen, newStale). The epoch pad keeps this below + // expiredClaimed. + return -now() } if ttl == 0 { return 0 } - return time.Now().Add(ttl).UnixNano() + return satAdd(now(), int64(ttl)) } func (o opt) apply(c *cfg) { o.fn(c) } @@ -151,23 +210,59 @@ func MaxAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxAge, c.a // after an entry has expired (so, total age is MaxAge + MaxStaleAge). A stale // value is the previous successfully cached value that is returned while the // value is being refreshed (a new value is being queried). As well, the stale -// value is returned while the refreshed value is erroring. This option is -// useless without MaxAge. +// value is returned while the refreshed value is erroring. Without MaxAge, +// stales still apply to manually Expired keys and to erroring refreshes. +// +// The stale window is anchored at the moment the value expired — its TTL +// (or idle extension) lapsing, its birth for values born expired +// (MaxAge(0)), or the Expire call that killed it — and every read agrees +// on it: TryGet serves the stale for exactly as long as a Get-driven +// refresh would, and a value whose window has closed is never served +// again. // -// A special value of -1 allows stale values to be returned indefinitely. +// A negative value (canonically -1) allows stale values to be returned +// indefinitely. func MaxStaleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxStaleAge = age }} } // MaxErrorAge sets the age to persist load errors. If not specified, the -// default is MaxAge. Using this option with 0 disables caching errors +// default is MaxAge — or the idle age, when only MaxIdleAge is set, since +// the idle age is then every entry's initial TTL (errors included, though an +// errored entry is never idle-extended). With neither MaxAge nor MaxIdleAge +// set, a load error is cached forever and never retried (until Set, Swap, +// Delete, or Expire). Using this option with 0 disables caching errors // entirely. +// +// A cached error suppresses repeat queries only when there is no stale +// value to return: if stale values are enabled and a valid stale exists, +// Get returns the stale and always drives a refresh, regardless of the +// error's remaining age. +// +// With errors uncached (age 0), concurrent Gets that collapsed onto a load +// that then errors do not share the error: only the Get that drove the +// load returns it, and each collapsed waiter re-drives its own load in +// turn, so one erroring wave of N collapsed Gets can issue up to N +// sequential loads. func MaxErrorAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxErrAge, c.errAgeSet = age, true }} } // MaxIdleAge opts in to extending an entry's expiry on each successful -// access. Each time Get or TryGet returns a Hit without an error, the +// access. Each time Get or TryGet returns a live Hit without an error, the // entry's expiry is reset to now + age. If MaxAge is not set, the idle -// age is also used as the initial TTL. +// age is also used as the initial TTL. A non-positive age is ignored +// entirely: no extension occurs, and no initial TTL is derived from it. +// The reset applies in both directions: with MaxIdleAge smaller than +// MaxAge, an access shortens the entry's remaining life from the +// MaxAge-given expiry to now + age. +// +// The reset is best effort under races: a concurrent Expire, or a Clean +// that has already deemed the entry expired, wins over an in-flight +// extension. In the other direction, an extension computed just before the +// entry's expiry passed can land just after it and briefly revive the entry +// for another idle window. The clock is re-sampled immediately before an +// extension is published, so the revival window is a few instructions wide +// (stretched only by the extending goroutine being descheduled exactly +// there), but it is not fully closed. func MaxIdleAge(age time.Duration) Opt { return opt{fn: func(c *cfg) { c.maxIdleAge = age }} } @@ -177,6 +272,12 @@ func MaxIdleAge(age time.Duration) Opt { // interval that is less than your MaxAge + MaxStaleAge. Values are only // candidates to be cleaned after the max age has elapsed. At worst, a value // may persist for a total of MaxAge + MaxStaleAge + AutoCleanInterval. +// +// The goroutine holds a reference to the cache, so a cache that started +// autocleaning is not garbage collected until StopAutoClean is called. +// +// If MaxStaleAge is negative (stales kept forever), Clean is a no-op and +// the goroutine is not started at all. func AutoCleanInterval(interval time.Duration) Opt { return opt{fn: func(c *cfg) { c.autoCleanInterval = interval }} } @@ -185,20 +286,25 @@ func AutoCleanInterval(interval time.Duration) Opt { // semantics. If you do not need to configure a cache at all, the zero value // cache is valid and usable. func New[K comparable, V any](opts ...Opt) *Cache[K, V] { - var cfg cfg + c := new(Cache[K, V]) + initCache(c, opts...) + return c +} + +// initCache applies options and starts the autoclean goroutine if +// configured. It initializes in place (rather than returning a new cache) +// so that NewItem and NewSet can initialize their embedded Cache: the +// autoclean goroutine closes over c, so the configured cache cannot be +// copied after this returns. +func initCache[K comparable, V any](c *Cache[K, V], opts ...Opt) { for _, opt := range opts { - opt.apply(&cfg) - } - c := &Cache[K, V]{ - cfg: cfg, - pd: new(loading[V]), + opt.apply(&c.cfg) } - c.pd.state.Store(statePromotingDelete) - if cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { + if c.cfg.autoCleanInterval > 0 && c.cfg.maxStaleAge >= 0 { c.quitClean = make(chan struct{}) go func() { - ticker := time.NewTicker(cfg.autoCleanInterval) + ticker := time.NewTicker(c.cfg.autoCleanInterval) defer ticker.Stop() for { select { @@ -210,195 +316,315 @@ func New[K comparable, V any](opts ...Opt) *Cache[K, V] { } }() } - return c } // Get returns the cache value for k, running the miss function in a goroutine -// if the key is not yet cached. If stale values are enabled, the currently -// cached value has an error, and there is an unexpired stale value, this -// returns the stale value and no error. +// if the key is not yet cached, the cached value has expired, or the cached +// value is an unexpired error with a valid stale to serve (a cached error +// suppresses repeat queries only while no stale exists; see MaxErrorAge). If +// stale values are enabled and an unexpired stale value exists — the currently +// cached value has an error, or it expired with its stale window still open — +// this returns the stale value and no error. +// +// The miss function runs on an internal goroutine: if it panics, the process +// crashes (the panic cannot be recovered by the Get caller); recover inside +// miss if you need to survive panics. A miss function must not call Get for +// the same key, in this goroutine or another: the inner Get would wait on +// the load the outer Get is driving and deadlock. func (c *Cache[K, V]) Get(k K, miss func() (V, error)) (v V, err error, s KeyState) { - r := c.read() - e := r.m[k] - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - return v, err, s - } - - // We missed in the read map. We lock and check again to guard against - // something concurrent. Odds are we are falling into the logic below. - c.mu.Lock() - r = c.read() - e = r.m[k] - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - c.mu.Unlock() - return v, err, s - } - - // We could have an entry in our read map that was deleted and has not - // yet gone through the promote&clear process. We only check the dirty - // map if the entry is nil. - if e == nil && r.incomplete { - e = c.dirty[k] - c.missed(r) - r = c.read() - if v, err, s = e.get(c.cfg.maxIdleAge); s == Hit { - c.mu.Unlock() + // Fast path: if the entry already exists and holds a live value, use + // it without any locking. + e := c.t.loadEntry(k) + if e != nil { + if v, err, s = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); s == Hit { return v, err, s } } - l := &loading[V]{ - stale: e.maybeNewStale(c.cfg.maxStaleAge), - } - l.wg.Add(1) - - // If we have no entry, this is completely new. If we had an entry, it - // was in the read map and not yet deleted through promotion. We know - // the pointer is not promotingDelete, since that is only set while - // promoting which requires the cache lock which we have right now. - // - // In the worst case we race with a concurrent Set and we override its - // results. - if e != nil { - e.p.Store(l) - } else { - e = new(ent[V]) - e.p.Store(l) - c.storeDirty(r, k, e) + // Slow path: acquire or create the entry, installing a fresh loading + // to drive a miss if there is no live value. A loading is always + // created before the entry that holds it is published, so a linked + // entry never holds a nil value slot; nil means the entry was + // tombstoned by Delete or Clean and is permanently dead (see entDel). + var l *loading[V] +outer: + for { + if e == nil { + l = &loading[V]{} + l.wg.Add(1) + var loaded bool + if e, loaded = c.t.loadOrStoreEntry(k, l); !loaded { + break // we created the entry, holding our loading + } + } + for { + prev := e.p.Load() + if prev == nil { + // Tombstoned. Unlink the dead entry (idempotent: the + // predicate re-checks under the bucket lock and leaves + // a re-created live entry alone) and start over with a + // fresh entry. + c.t.deleteEntryIf(k, entDead[K, V]) + e = nil + continue outer + } + // There is an existing value. Use entGet to decide whether + // to wait for it (finalized hit), return a stale (load in + // flight with a valid stale), or replace it (finalized but + // expired or errored). + if v, err, s = entGet(e, c.cfg.maxIdleAge, c.cfg.maxStaleAge); s == Hit { + return v, err, s + } + if s == Stale && !prev.finalized() && e.p.Load() == prev { + // A load is already in flight and its caller will + // finalize it; piggyback and return the stale now. The + // slot re-check pins the stale to prev: entGet loads + // the slot itself, so without it the stale could come + // from a newer loading while the in-flight test + // inspects prev. + return v, err, s + } + // Replace prev only if it is finalized and expired or + // errored. entGet's non-Hit result usually implies that, + // but entGet re-loads the slot itself: a racing writer can + // finalize prev live (or replace it) between our load of + // prev and entGet's. Re-deriving from prev keeps us from + // demoting a freshly finalized live value to a stale and + // driving a redundant load; the retry re-reads the slot. + if !prev.finalized() || (prev.err == nil && !prev.expired(now())) { + continue + } + // prev is finalized and expired or errored, possibly with a + // valid stale that nothing is refreshing (if it was in + // flight with no stale, entGet waited for it to finalize). + // Install a fresh loading carrying a stale snapshot of prev + // and drive a new load. The CAS fails if another goroutine + // got here first; re-evaluate what it installed. + l = &loading[V]{ + stale: entMaybeNewStale(e, c.cfg.maxStaleAge), + } + l.wg.Add(1) + if e.p.CompareAndSwap(prev, l) { + break outer + } + } } - c.mu.Unlock() go func() { v, err := miss(); l.setve(v, err, c.cfg.newExpires(err)) }() - // We could have set our own stale value which can be returned - // immediately rather than waiting for the get. If there is a valid - // stale to be returned now, `get` returns it. If `get` returns a Hit, - // we ended up waiting for ourself and we return a miss. There should - // be no Miss returns from `get`. - v, err, s = e.get(c.cfg.maxIdleAge) + // Wait for our loading to finalize, then return the value. Waiting via + // l.wg (our own) synchronizes with whoever called wg.Done — the miss + // goroutine's setve or a racing Swap's defer — even if e.p has since + // been replaced by another writer. extend=0: this Get drove the load + // and publicly returns Miss, so it is not an idle-extending access + // (MaxIdleAge documents extension on Hits); the value keeps the + // initial TTL it was born with. staleAge=0: this Get returns the + // load's result, never a re-derivation of that result as a Stale (a + // short-TTL value can expire the instant it loads; its derived window + // serves later readers, not the load that produced it) — the only + // stale surfaced here is a refresh companion carried by a loading + // still in flight. + v, err, s = entGet(e, 0, 0) switch s { case Miss, Hit: - // We always return l.v and l.err. If this expires immediately, - // get may return no value. We want to return at least the - // value generated from this miss. + l.wg.Wait() return l.v, l.err, Miss } return v, err, Stale } -func (c *Cache[K, V]) tryLoadEnt(k K, dirty func()) *ent[V] { - r := c.read() - e := r.m[k] - if e == nil && r.incomplete { - c.mu.Lock() - r = c.read() - e = r.m[k] - if e == nil && r.incomplete { - e = c.dirty[k] - if dirty != nil { - dirty() - } - c.missed(r) - } - c.mu.Unlock() - } - return e -} - // TryGet returns the value for the given key if it is cached. This returns -// either the currently stored value, or the current stale if the load errored, -// or the load error if there is no stale. If nothing is cached, or what is -// cached is expired, this returns Miss. -func (c *Cache[K, V]) TryGet(k K) (V, error, KeyState) { - e := c.tryLoadEnt(k, nil) - return e.tryGet(0, c.cfg.maxIdleAge) +// either the currently stored value, or the current stale if the latest load +// errored or the value has expired while its stale is still valid, or the +// load error if there is no stale. If nothing is cached, or what is cached is +// expired with no valid stale, this returns Miss. +func (c *Cache[K, V]) TryGet(k K) (v V, err error, _ KeyState) { + e := c.t.loadEntry(k) + if e == nil { + return v, err, Miss + } + return entTryGet(e, 0, c.cfg.maxIdleAge, c.cfg.maxStaleAge) } // Delete deletes the value for a key and returns the prior value, if stored // (i.e. the return from TryGet). -func (c *Cache[K, V]) Delete(k K) (V, error, KeyState) { - e := c.tryLoadEnt(k, func() { delete(c.dirty, k) }) - defer e.del() - return e.tryGet(0, 0) +// +// Delete physically removes the entry from the trie. If a concurrent Get is +// holding a reference to the deleted entry via its in-flight loading, that +// Get still completes with the miss function's result; a new Get arriving +// after Delete creates a fresh entry and may drive an independent miss. A +// load that finalizes concurrently with the removal may be reported as the +// removed value even though no read ever observed it cached. +func (c *Cache[K, V]) Delete(k K) (v V, err error, _ KeyState) { + e := c.t.loadEntry(k) + if e == nil { + return v, err, Miss + } + // Tombstone the value slot first; the captured loading is exactly what + // this Delete removed, so the returned value is the value removed. + // Then physically unlink the entry from the trie so the key (and any + // memory it refers to) is eligible for GC. The nil slot is terminal + // (see entDel), so entDead observed under the bucket lock is stable; a + // concurrent Swap or Get that finds the tombstone re-creates the key + // as a fresh entry, which the predicate leaves alone. + // + // The clock is sampled before the tombstone CAS so the removed value + // is classified (live, stale, or expired) as of the removal itself; + // sampling after would let a value that expires in between be + // misreported as already gone even though this Delete removed it live. + n := now() + was := entDel(e) + c.t.deleteEntryIf(k, entDead[K, V]) + if was == nil { + return v, err, Miss + } + return loadingTryGet(was, n, 0, c.cfg.maxStaleAge) } // Expire sets a stored value to expire immediately, meaning the next Get will // be a miss. If stale values are enabled, the next Get will trigger the miss // function but still allow the now-stale value to be returned. +// +// Expire only affects entries that have finished loading. Calling Expire for a +// key whose miss function is still running is a no-op; the in-flight load will +// complete with its normal TTL and is not canceled or shortened. Expiring an +// already-expired entry is also a no-op: the expiry is never moved forward, +// so a dead value's stale window cannot be re-anchored or revived, and a +// concurrent Clean's claim is never overwritten. (An Expire that races the +// entry's natural death can land just after it and trim the dead value's +// remaining stale window; the window only ever shrinks, never grows.) func (c *Cache[K, V]) Expire(k K) { - e := c.tryLoadEnt(k, nil) - if l := e.load(); l != nil && l.finalized() { - l.expires.Store(now() - 1) + e := c.t.loadEntry(k) + if e == nil { + return + } + l := entLoad(e) + if l == nil || !l.finalized() { + return + } + // CAS rather than a blind store, and only ever backward in time. A + // blind store would move an already-expired word forward — re-anchoring + // the dead value's stale window at the Expire (resurrecting data the + // caller explicitly invalidated) and clobbering Clean's claim sentinel + // inside its claim window. The loop still beats a racing idle + // extension: if the extension's CAS lands first, ours fails, reloads + // the extended (live) expiry, and expires it. + for { + cur := l.expires.Load() + n := now() + if cur != 0 && cur <= n { + return // already expired (including born-expired and claimed) + } + if l.expires.CompareAndSwap(cur, n-1) { + return + } } } // Range calls fn for every cached value. If fn returns false, iteration stops. +// +// Range does not necessarily correspond to any consistent snapshot of the +// cache: no key is visited more than once, but if a key is set, deleted, or +// expires concurrently (including from fn itself), Range may reflect any +// state of that key from any point during the call. Range does not block +// other methods on the cache; even fn may call them. func (c *Cache[K, V]) Range(fn func(K, V, error) bool) { // When ranging, repeated time.Now() calls add up, so we get the // current time when we enter range and avoid it in all tryGet calls. - now := now() - c.each(func(k K, e *ent[V]) bool { - v, err, s := e.tryGet(now, 0) + tn := now() + c.t.walk(func(e *ent[K, V]) bool { + v, err, s := entTryGet(e, tn, 0, c.cfg.maxStaleAge) if s.IsMiss() { return true } - return fn(k, v, err) + return fn(e.key, v, err) }) } -// Similar to sync.Map, we promote on range because range is O(N) (usually) and -// this amortizes out. -func (c *Cache[K, V]) each(fn func(K, *ent[V]) bool) { - r := c.read() - if r.incomplete { - c.mu.Lock() - r = c.read() - if r.incomplete { - c.promote() - r = c.read() - } - c.mu.Unlock() - } - for k, e := range r.m { - if !fn(k, e) { - return - } - } -} - // Clean deletes all expired values from the cache. A value is expired if // MaxAge is used and the entry is older than the max age, or if you manually -// expired a key. If MaxStaleAge is used and not -1, the entry must be older -// than MaxAge + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. +// expired a key. An entry is removed only once it is invisible to every +// read: with MaxStaleAge used and not negative, a value's stale window runs +// until MaxAge + MaxStaleAge — anchored at the birth for entries born +// expired (MaxAge(0) or MaxErrorAge(0)) — and an entry whose window is +// still open is never removed. Expired errors are removed as soon as no +// valid stale remains. If MaxStaleAge is negative, Clean returns +// immediately. +// +// Clean also physically removes entries that were tombstoned by prior Delete +// calls, so the trie's memory footprint does not grow unboundedly with +// delete churn. func (c *Cache[K, V]) Clean() { - // If MaxStaleAge is -1, the user opted into persisting stales forever - // and we do not clean. if c.cfg.maxStaleAge < 0 { return } - now := now() - c.each(func(k K, e *ent[V]) bool { - if l := e.load(); l != nil && l.finalized() { - expires := l.expires.Load() - if expires != 0 && now > expires+int64(c.cfg.maxStaleAge) { - c.Delete(k) + tn := now() + + // First pass: tombstone expired entries. Collect keys that need + // physical removal (both newly-tombstoned and pre-existing tombstones). + var toPrune []K + c.t.walk(func(e *ent[K, V]) bool { + l := entLoad(e) + if l == nil { + toPrune = append(toPrune, e.key) + return true + } + if !l.finalized() { + return true + } + expires := l.expires.Load() + // An entry is reclaimable only once it is invisible to every + // read, i.e. exactly when TryGet would return Miss: the value + // must be expired, any carried refresh-companion stale (servable + // while the result is an error) must be out of its window, and an + // err-free value's derived self-stale window must be closed. + // staleOpen mirrors the read paths, anchoring born-expired + // entries at their birth — so dead stale-less entries (error + // churn under MaxErrorAge(0)) are reclaimed immediately, and a + // huge MaxStaleAge cannot block their reclaim forever. Errored + // values have no self-stale: they are reclaimable as soon as + // they are expired with no valid companion. + expired := expires != 0 && expires <= tn + if expired && (l.stale == nil || l.stale.expired(tn)) && + (l.err != nil || !staleOpen(expires, tn, c.cfg.maxStaleAge)) { + // Win the expiry word before tombstoning: idle extension + // CASes against the expiry it observed, so claiming the + // word here makes a racing extension fail — the same way it + // loses to Expire — rather than be silently thrown away by + // our eviction. If instead the extension wins, the entry is + // in use; leave it for a later pass. + // + // The claimed word also reads as "stale window certified + // closed" everywhere (staleOpen, entMaybeNewStale), so a + // racing Get that snapshots this loading between our two + // CASes can never re-derive a window for the dead value. + if l.expires.CompareAndSwap(expires, expiredClaimed) && e.p.CompareAndSwap(l, nil) { + toPrune = append(toPrune, e.key) } } return true }) + + // Second pass: physically unlink the tombstoned entries. The nil value + // slot is terminal (see entDel), so the predicate observing nil under + // the bucket lock cannot be invalidated by a concurrent writer; if the + // key was deleted and re-created in the meantime, the lookup finds the + // fresh live entry and the predicate leaves it alone. + for _, k := range toPrune { + c.t.deleteEntryIf(k, entDead[K, V]) + } } -// Clear deletes all keys from the cache, resetting it to an empty state. +// Clear deletes all keys from the cache, resetting it to an empty state. A +// load in flight at the time of the Clear still completes and returns its +// result to the Gets waiting on it, but the result is not cached: a later +// Get for the same key misses and drives a fresh, independent load. func (c *Cache[K, V]) Clear() { - c.mu.Lock() - c.dirty = nil - c.storeRead(read[K, V]{}) - c.misses = 0 - c.mu.Unlock() + c.t.clear() } -// StopAutoClean stops the auto clean goroutine that is began with the -// AutoClean option. +// StopAutoClean stops the auto clean goroutine that is begun with the +// AutoCleanInterval option. func (c *Cache[K, V]) StopAutoClean() { c.quitOnce.Do(func() { if c.quitClean != nil { @@ -408,26 +634,34 @@ func (c *Cache[K, V]) StopAutoClean() { } // Swap sets a value for a key. If the key is currently loading via Get, the -// load is canceled and Get returns the value from Swap. This returns the -// previously stored value, or the previous stale if the load errored, or the -// previous error if there is no stale. If nothing was cached, this returns -// Miss. +// load's result is discarded (the miss function itself is not interrupted) +// and Get returns the value from Swap. This returns the previously stored +// value, or the previous stale if the load errored, or the previous error if +// there is no stale. If nothing was cached, this returns Miss. +// +// A load that finalizes concurrently with the swap may be reported as the +// prior value even though no read ever observed it cached. func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { l := c.finalizedLoading(v) var was *loading[V] + var wasN int64 defer func() { if was == nil { return } // The following block is similar to setve, but expanded to - // return our prior value (or stale, or err) if it exists. - now := now() + // return our prior value (or stale, or err) if it exists. wasN + // was sampled just before the CAS that displaced was, so the + // displaced value is classified (live, stale, or expired) as of + // the swap itself; a fresh sample here could misreport a value + // that expired between the CAS and this defer. + n := wasN if !was.finalized() { was.mu.Lock() if !was.finalized() { - if was.stale != nil && !was.stale.expired(now) { + if was.stale != nil && !was.stale.expired(n) { old, oldState = was.stale.v, Stale } was.v = v @@ -439,44 +673,52 @@ func (c *Cache[K, V]) Swap(k K, v V) (old V, oldErr error, oldState KeyState) { } was.mu.Unlock() } - if expired := was.expired(now); was.err != nil || expired { - if was.stale != nil && !was.stale.expired(now) { - old, oldState = was.stale.v, Stale - } - if expired { - return + // The displaced value is reported exactly as a TryGet at wasN + // would have reported it: classifyFinalized is the one + // classification every read shares. + old, oldErr, oldState = classifyFinalized(was, was.expires.Load(), n, c.cfg.maxStaleAge) + }() + + // Fast path: if the entry already exists with a live value, CAS the + // value slot directly without going through the trie's bucket lock. A + // nil slot means the entry was tombstoned by Delete or Clean; it is + // never written again (see entDel), so we fall to the slow path to + // re-create the key rather than writing into the dead entry. + if e := c.t.loadEntry(k); e != nil { + if rm := e.p.Load(); rm != nil { + was, wasN = rm, now() + if e.p.CompareAndSwap(rm, l) { + return old, oldErr, oldState } + // Lost the CAS; fall into the slow path. } - old, oldErr, oldState = was.v, was.err, Hit - }() + } - r := c.read() - if e, ok := r.m[k]; ok { + // Slow path: get or create an entry and swap our value in. The trie's + // bucket lock makes creation exclusive; tombstoned entries are + // unlinked and the key re-created as a fresh entry, never resurrected + // in place. + for { + e, loaded := c.t.loadOrStoreEntry(k, l) + if !loaded { + // We are the entry's installer; nothing was there before. + was = nil + return old, oldErr, oldState + } for { rm := e.p.Load() - if rm != nil && rm.promotingDelete() { // deleted & currently being ignored in a promote + if rm == nil { + // Tombstoned by Delete or Clean. Unlink the dead entry + // and retry with a fresh one. + c.t.deleteEntryIf(k, entDead[K, V]) break } - was = rm + was, wasN = rm, now() if e.p.CompareAndSwap(rm, l) { return old, oldErr, oldState } } } - - c.mu.Lock() - r = c.read() - if e := r.m[k]; e != nil { - was = e.p.Swap(l) // was not in read, but promoted by the time we entered the lock and is now in read - } else if e = c.dirty[k]; e != nil { - was = e.p.Swap(l) - } else { - e := new(ent[V]) - e.p.Store(l) - c.storeDirty(r, k, e) - } - c.mu.Unlock() - return old, oldErr, oldState } func (l *loading[V]) setve(v V, err error, expires int64) { @@ -495,309 +737,411 @@ func (l *loading[V]) setve(v V, err error, expires int64) { } // Set sets a value for a key. If the key is currently loading via Get, the -// load is canceled and Get returns the value from Set. +// load's result is discarded (the miss function itself is not interrupted) +// and Get returns the value from Set. func (c *Cache[K, V]) Set(k K, v V) { c.Swap(k, v) } // CompareAndSwap swaps the old and new values for k if the value has finished -// loading and the value is equal to old. The type V must be comparable. +// loading without an error, is not expired, and is equal to old. The type V +// must be comparable. Stale values are not considered: only the live value +// is compared against. +// +// The expiry check is best effort: a value that expires while the swap is +// in flight — its TTL lapsing or a concurrent Expire landing — may still +// be swapped. func (c *Cache[K, V]) CompareAndSwap(k K, old, new V) bool { - r := c.read() - if e, ok := r.m[k]; ok { - return c.tryCAS(e, old, new, true) - } else if !r.incomplete { + e := c.t.loadEntry(k) + if e == nil { return false } - c.mu.Lock() - defer c.mu.Unlock() - r = c.read() - if e, ok := r.m[k]; ok { - return c.tryCAS(e, old, new, true) - } else if e, ok := c.dirty[k]; ok { - defer c.missed(r) - return c.tryCAS(e, old, new, true) - } - return false + return c.tryCAS(e, old, new, true) } // CompareAndDelete deletes the entry for k if the value has finished loading -// and the value is equal to old. The type V must be comparable. -func (c *Cache[K, V]) CompareAndDelete(k K, old V) (deleted bool) { - r := c.read() - e, ok := r.m[k] - if !ok && r.incomplete { - c.mu.Lock() - r = c.read() - e, ok = r.m[k] - if !ok && r.incomplete { - e, ok = c.dirty[k] - c.missed(r) - } - c.mu.Unlock() +// without an error, is not expired, and is equal to old. The type V must be +// comparable. Stale values are not considered: only the live value is +// compared against. +// +// The expiry check is best effort: a value that expires while the delete +// is in flight — its TTL lapsing or a concurrent Expire landing — may +// still be deleted. +func (c *Cache[K, V]) CompareAndDelete(k K, old V) bool { + e := c.t.loadEntry(k) + if e == nil { + return false } - if ok { - return c.tryCAS(e, old, old, false) + if !c.tryCAS(e, old, old, false) { + return false } - return false + // The successful CAS tombstoned the value slot; physically unlink the + // entry like Delete does, so delete churn does not leak entries. + c.t.deleteEntryIf(k, entDead[K, V]) + return true } -func (c *Cache[K, V]) tryCAS(e *ent[V], old, new V, useNew bool) bool { +func (c *Cache[K, V]) tryCAS(e *ent[K, V], old, new V, useNew bool) bool { l := e.p.Load() - if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { + if !casMatches(l, old) { return false } var l2 *loading[V] if useNew { l2 = c.finalizedLoading(new) } + if casPublishHook != nil { + casPublishHook() + } for { + // Re-validate liveness immediately before publishing: the + // allocation above is the widest deschedule risk in this call, and + // a value whose TTL lapsed while we were parked there must not be + // matched — TryGet already reports it as Miss. The handful of + // instructions between this check and the CAS remain best effort + // (see CompareAndSwap). + if !casMatches(l, old) { + return false + } if e.p.CompareAndSwap(l, l2) { return true } l = e.p.Load() - if l == nil || !l.onlyFinalized() || any(l.v) != any(old) { - return false - } } } -////////////////////// -// CACHE READ/DIRTY // -////////////////////// - -func (c *Cache[K, V]) read() read[K, V] { - p := c.r.Load() - if p == nil { - return read[K, V]{} - } - return *p -} -func (c *Cache[K, V]) storeRead(r read[K, V]) { c.r.Store(&r) } - -func (c *Cache[K, V]) storeDirty(r read[K, V], k K, e *ent[V]) { - if !r.incomplete { - if c.dirty == nil { - c.dirty = make(map[K]*ent[V]) - } - c.storeRead(read[K, V]{m: r.m, incomplete: true}) - } - c.dirty[k] = e -} - -func (c *Cache[K, V]) missed(r read[K, V]) { - c.misses++ - if len(c.dirty) == 0 || c.misses > len(r.m)>>1 { - c.promote() - } -} - -func (c *Cache[K, V]) promote() { - r := c.read() - keep := r.m - - defer func() { - c.storeRead(read[K, V]{m: keep}) - c.misses = 0 - }() - - if len(c.dirty) == 0 { - return - } - - keep = make(map[K]*ent[V], len(keep)+len(c.dirty)) - for k, e := range c.dirty { - keep[k] = e - } - c.dirty = nil - -outer: - for k, e := range r.m { - p := e.p.Load() - for p == nil { - if e.p.CompareAndSwap(nil, c.pd) { - continue outer - } - p = e.p.Load() // concurrently deleted while promoting - } - if p != c.pd { - keep[k] = e - } +// casMatches reports whether l holds a live cached value equal to old: +// finalized, not errored, not expired. CompareAndSwap and CompareAndDelete +// must agree with TryGet about whether a value exists — an expired or +// errored entry is not a value that can be compared against (an errored +// loading's v is whatever the miss function returned beside the error, +// which was never cached as a value). +// +// Liveness is judged on a coherent (expiry word, clock) pair from +// expiresNow, exactly as the read paths judge it: the clock postdates the +// word it is paired with, so a kill that completed before the pair was +// taken — a lapsed TTL, an Expire, a Clean claim — is always seen, no +// matter how long this call was descheduled mid-check, while an idle +// extension racing the pair re-derives rather than failing a +// continuously-live match. The pair still cannot be made atomic with the +// publishing CAS: a successful swap takes effect at the pointer CAS, and +// the clock cannot be read at that exact instant. tryCAS re-validates +// immediately before each CAS attempt; the instructions-wide residue +// between that check and the CAS is documented on CompareAndSwap and +// CompareAndDelete. +func casMatches[V any](l *loading[V], old V) bool { + if l == nil || !l.finalized() || l.err != nil || any(l.v) != any(old) { + return false } + expires, n := l.expiresNow() + return expires == 0 || expires > n } -///////// -// ENT // -///////// +/////////////////// +// ENTRY HELPERS // +/////////////////// func (c *Cache[K, V]) finalizedLoading(v V) *loading[V] { + // No stale companion: the companion is only ever read while a loading + // is in flight or errored, and a Set/Swap-created loading is born + // finalized with no error. The value's own post-expiry stale window is + // derived from the expires word at read time (see staleOpen), so it + // needs no snapshot here either. l := &loading[V]{ v: v, } - expires := c.cfg.newExpires(nil) - if expires != 0 || c.cfg.maxStaleAge != 0 { + if expires := c.cfg.newExpires(nil); expires != 0 { l.expires.Store(expires) - if c.cfg.maxStaleAge != 0 { - l.stale = newStale(v, l.expires.Load(), c.cfg.maxStaleAge) - } } l.state.Store(stateFinalized) return l } -func (l *loading[V]) promotingDelete() bool { return l.state.Load() == 2 } -func (l *loading[V]) finalized() bool { return l.state.Load() != 0 } -func (l *loading[V]) onlyFinalized() bool { return l.state.Load() == 1 } +func (l *loading[V]) finalized() bool { return l.state.Load() != 0 } -func (e *ent[V]) del() { - if e == nil { - return - } +// entDel tombstones the entry by atomically setting its value slot to nil, +// returning the loading it displaced (nil if the entry was already +// tombstoned). +// +// A nil value slot is terminal: no code path ever stores a non-nil loading +// over it. Writers that encounter a tombstone unlink the dead entry +// (deleteEntryIf with entDead) and re-create the key as a fresh entry. This +// is what makes physical unlinking sound: entDead evaluated under the +// trie's bucket lock cannot be invalidated by a concurrent lock-free CAS, +// so an unlinked entry is always dead and a live entry is never unlinked. +func entDel[K comparable, V any](e *ent[K, V]) *loading[V] { for { p := e.p.Load() - if p == nil || p.promotingDelete() { - return + if p == nil { + return nil } if e.p.CompareAndSwap(p, nil) { - return + return p } } } -func (e *ent[V]) load() *loading[V] { - if e == nil { - return nil - } - p := e.p.Load() - if p == nil || p.promotingDelete() { - return nil - } - return p +// entDead reports whether the entry is tombstoned. Used as a deleteEntryIf +// predicate: nil value slots are terminal (see entDel), so a nil observed +// under the bucket lock is stable. +func entDead[K comparable, V any](e *ent[K, V]) bool { return e.p.Load() == nil } + +// entLoad returns the entry's current loading, or nil if the slot is +// tombstoned. Callers guarantee e is non-nil. +func entLoad[K comparable, V any](e *ent[K, V]) *loading[V] { + return e.p.Load() } -func (l *loading[V]) expired(now int64) bool { +func (l *loading[V]) expired(n int64) bool { expires := l.expires.Load() - return expires != 0 && expires <= now // 0 means either miss not resolved, or no max age + return expires != 0 && expires <= n // 0 means either miss not resolved, or no max age } -func (s *stale[V]) expired(now int64) bool { +func (s *stale[V]) expired(n int64) bool { expires := s.expires - return expires != 0 && expires <= now + return expires != 0 && expires <= n } -// If an entry is expiring, we create a new entry with this previous entry as -// a stale. +// entMaybeNewStale derives a stale snapshot from the entry's current +// loading, for use as the refresh companion of a replacement loading. // -// - if entry is nil, no stale, return nil +// - if entry has no live loading, no stale, return nil // - if no stale age, we are not using stales, return nil -// - if entry has an error, return prior stale +// - if the loading is still pending, return nil (no value to snapshot) +// - if loading has an error, return the prior stale (companion passthrough) +// - if Clean claimed the loading (window certified closed), return nil // - if age is < 0, return new unexpiring stale -// - else, return new stale with prior expiry + stale age -func (e *ent[V]) maybeNewStale(age time.Duration) *stale[V] { - if e == nil || age == 0 { +// - else, return a stale spanning the value's derived window: its expiry +// (or birth, for born-expired values) plus the stale age — the same +// window staleOpen computes on the read paths +func entMaybeNewStale[K comparable, V any](e *ent[K, V], age time.Duration) *stale[V] { + if age == 0 { return nil } - l := e.load() + l := entLoad(e) if l == nil || !l.finalized() { return nil } if l.err != nil { return l.stale } - return newStale(l.v, l.expires.Load(), age) + expires := l.expires.Load() + if expires == expiredClaimed { + // Clean claimed this loading: the value is past its full window, + // which Clean certified closed. Never re-derive a window for it. + return nil + } + return newStale(l.v, expires, age) +} + +// staleOpen reports whether the derived self-stale window of an expired, +// err-free loading is still open at n: the same window newStale builds +// when the value is snapshotted as a refresh companion, so every read +// agrees with every refresh about how long an expired value remains +// servable. expires must be the loading's non-zero expiry word: positive +// is the expiry itself, a negated birth anchors born-expired values at +// their birth, and a Clean-claimed word means the window was certified +// closed. +func staleOpen(expires, n int64, age time.Duration) bool { + if age == 0 || expires == expiredClaimed { + return false + } + if age < 0 { + return true + } + anchor := expires + if anchor < 0 { + anchor = -anchor // born expired: the negated birth (see newExpires) + } + return n < satAdd(anchor, int64(age)) } -// Actually returns the stale; age must be non-zero. +// newStale returns a fresh stale; age must be non-zero. +// +// expires is the main entry's expiry word. A negative word carries the +// value's negated birth (see newExpires) and anchors the window at the +// birth, matching staleOpen. A 0 word (the value never expires) anchors at +// now; it is reachable only when a racing writer replaced the slot between +// the caller's expiry check and our snapshot, and the caller's install CAS +// then fails and discards the snapshot — but handle it sanely regardless. +// Callers must filter expiredClaimed before calling: a claimed value's +// window is certified closed and must not be re-derived (see +// entMaybeNewStale, staleOpen). func newStale[V any](v V, expires int64, age time.Duration) *stale[V] { if age < 0 { return &stale[V]{v: v} } - return &stale[V]{v, expires + int64(age)} + switch { + case expires == 0: + expires = now() + case expires < 0: + expires = -expires + } + return &stale[V]{v, satAdd(expires, int64(age))} } -// get always returns the value or the stale value. We do not check if our -// value is expired: we call this at the end of Get, we must always return -// something even if it is to be immediately expired. -func (e *ent[V]) get(extend time.Duration) (v V, err error, state KeyState) { - l := e.load() - var waited bool - if l == nil { - // Could be nil if deleted while in the read map, or - // promotingDelete. - return v, err, state +// expiresPairingHook, if non-nil, runs in expiresNow between the expiry +// load and the clock sample, so tests can deterministically hold a reader +// descheduled in that window. Never set in production code. +var expiresPairingHook func() + +// casPublishHook, if non-nil, runs in tryCAS between the replacement +// loading's allocation and the pre-publish liveness re-validation, so tests +// can deterministically hold a CAS caller descheduled in that window. Never +// set in production code. +var casPublishHook func() + +// expiresNow returns the loading's expiry word and a clock sample forming a +// coherent pair. The word is loaded before the clock is sampled: Expire +// CASes in now()-1 from its own clock, which can be ahead of a clock +// sampled before our load, so sampling after the load guarantees a +// concurrent Expire is classified as expired rather than extended over. The +// word is then confirmed after the sample: an idle extension landing +// between the two would otherwise pair the dead pre-extension word with a +// fresh clock, misreporting an entry that is live for this entire call as +// Stale or Miss. Each retry re-loads before re-sampling, preserving the +// Expire ordering. +func (l *loading[V]) expiresNow() (expires, n int64) { + expires = l.expires.Load() + if expiresPairingHook != nil { + expiresPairingHook() } - if !l.finalized() { - if l.stale != nil && !l.stale.expired(now()) { - return l.stale.v, nil, Stale + for { + n = now() + if again := l.expires.Load(); again != expires { + expires = again + continue } - l.wg.Wait() - waited = true + return expires, n } +} - // If we did not wait and our entry is expired (value or error), or if - // our entry is not expired but has errored, we potentially return the - // stale entry. - // - // If we waited, we could immediately be expired due to time sync, or - // if the user is configured to never cache and they're just using - // request collapsing: we still want to return the now expired value. - now := now() - if (!waited && l.expired(now)) || l.err != nil { - if l.stale != nil && !l.stale.expired(now) { +// classifyFinalized reports a finalized loading exactly as a read at clock +// n does: a valid companion stale (the previous generation) outranks an +// error; with no companion, an unexpired error is itself the cached result +// and a dead error is a miss; an expired err-free value is served as Stale +// while its derived window is open (see staleOpen) and is a miss after. +// entGet, loadingTryGet, and Swap's displaced-value report all share this +// classification, so every read and every report agree by construction. +func classifyFinalized[V any](l *loading[V], expires, n int64, staleAge time.Duration) (v V, err error, state KeyState) { + expired := expires != 0 && expires <= n + if l.err != nil { + if l.stale != nil && !l.stale.expired(n) { return l.stale.v, nil, Stale } - // The stale value is expired: if our entry is not expired, - // this must be an error we waited on. - if !l.expired(now) { + if !expired { return l.v, l.err, Hit } return v, err, state } - if extend > 0 && l.err == nil { - l.expires.Store(now + int64(extend)) + if expired { + if staleOpen(expires, n, staleAge) { + return l.v, nil, Stale + } + return v, err, state } return l.v, l.err, Hit } -func (e *ent[V]) tryGet(n64 int64, extend time.Duration) (v V, err error, state KeyState) { - if e == nil { +// entGet returns the value, the stale value, or — after waiting on an +// in-flight load — whatever the load produced. When we waited, expiry alone +// does not force a miss: Get must hand its caller something even if the +// value expired the instant it loaded (e.g. request collapsing with +// MaxAge(0)). +func entGet[K comparable, V any](e *ent[K, V], extend, staleAge time.Duration) (v V, err error, state KeyState) { + l := entLoad(e) + var waited bool + if l == nil { return v, err, state } - l := e.load() - if l == nil { // deleting or promotedDelete - return v, err, state + if !l.finalized() { + if l.stale != nil && !l.stale.expired(now()) { + return l.stale.v, nil, Stale + } + l.wg.Wait() + waited = true } - // Fast path: finalized, no expiry, no error — skip time checks. - if l.onlyFinalized() && l.expires.Load() == 0 && l.err == nil { + + expires, n := l.expiresNow() + expired := expires != 0 && expires <= n + if waited && l.err == nil && expired { + // We waited and the load finalized already expired (MaxAge(0) + // collapsing, or a TTL shorter than the load itself): hand back + // the now-expired value as a courtesy rather than re-drive. The + // courtesy is not an idle-extending access; the value is + // returned, not resurrected. return l.v, nil, Hit } - if n64 == 0 { - n64 = now() + v, err, state = classifyFinalized(l, expires, n, staleAge) + if extend > 0 && state == Hit && l.err == nil { + // Extend the idle expiry via CAS against the expiry we evaluated + // above so that a concurrent Expire (or Swap finalization, or a + // Clean eviction) is not silently overwritten; if expires + // changed, the other writer wins. An err-free Hit is structurally + // unexpired at n (a waited-on load that finalized already expired + // was returned above as a courtesy, never reaching here), but n + // may have gone stale if we were descheduled since it was + // sampled: a CAS landing after the expiry passed would revive an + // entry that concurrent readers may have already reported as + // Miss. Re-sample the clock immediately before publishing; the + // instructions between this sample and the CAS remain best + // effort (see MaxIdleAge). + if now() < expires { + l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) + } + } + return v, err, state +} + +func entTryGet[K comparable, V any](e *ent[K, V], n64 int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { + l := entLoad(e) + if l == nil { + return v, err, state } - now := n64 + return loadingTryGet(l, n64, extend, staleAge) +} +// loadingTryGet is entTryGet for a loading already plucked from an entry; +// Delete uses it directly on the loading it captured while tombstoning, so +// the value it returns is exactly the value it removed. +func loadingTryGet[V any](l *loading[V], n64 int64, extend, staleAge time.Duration) (v V, err error, state KeyState) { + // Fast path: finalized, no expiry, no error — skip time checks. + if l.finalized() && l.expires.Load() == 0 && l.err == nil { + return l.v, nil, Hit + } // If we are loading but there is a valid stale, return it, otherwise // return immediately: no get. if !l.finalized() { - if l.stale != nil && !l.stale.expired(now) { + if n64 == 0 { + n64 = now() + } + if l.stale != nil && !l.stale.expired(n64) { return l.stale.v, nil, Stale } return v, err, state } - // If we have an error or we are expired, we maybe return the stale. - if expired := l.expired(now); l.err != nil || expired { - if l.stale != nil && !l.stale.expired(now) { - return l.stale.v, nil, Stale - } - if expired { - return v, err, state + var expires int64 + if n64 == 0 { + expires, n64 = l.expiresNow() + } else { + // A caller-provided n64 (Range's batch timestamp, Delete's + // pre-removal clock) predates this load of the word, so the + // stale-word hazard expiresNow guards against cannot arise; the + // caller pins its snapshot to n64 and never extends. + expires = l.expires.Load() + } + n := n64 + v, err, state = classifyFinalized(l, expires, n, staleAge) + if extend > 0 && state == Hit && l.err == nil { + // CAS so a concurrent Expire is not silently overwritten; see the + // matching comment in entGet, including the pre-publish clock + // re-sample. + if now() < expires { + l.expires.CompareAndSwap(expires, satAdd(n, int64(extend))) } } - if extend > 0 && l.err == nil { - l.expires.Store(now + int64(extend)) - } - return l.v, l.err, Hit + return v, err, state } ////////// @@ -815,29 +1159,27 @@ type Item[V any] struct { // semantics. If you do not need to configure an item at all, the zero value // item is valid and usable. func NewItem[V any](opts ...Opt) *Item[V] { - var c cfg - for _, opt := range opts { - opt.apply(&c) - } - return &Item[V]{ - c: Cache[struct{}, V]{ - cfg: c, - }, - } + i := new(Item[V]) + initCache(&i.c, opts...) + return i } // Get returns the currently cached value, running the miss function in a -// goroutine if the item is not yet cached. If stale values are enabled, the -// currently cached value has an error, and there is an unexpired stale value, -// this returns the stale value and no error. +// goroutine if the item is not yet cached, the cached value has expired, or +// the cached value is an unexpired error with a valid stale to serve. If +// stale values are enabled and an unexpired stale value exists — the +// currently cached value has an error, or it expired with its stale window +// still open — this returns the stale value and no error. See Cache.Get for +// the miss function's panic and re-entrancy caveats. func (i *Item[V]) Get(miss func() (V, error)) (v V, err error, state KeyState) { return i.c.Get(struct{}{}, miss) } -// TryGet returns the value the item if it is cached. This returns either the -// currently loaded value, or the current stale if the load errored, or the -// load error if there is no stale. If nothing is cached, or what is cached is -// expired, this returns Miss. +// TryGet returns the item's value if it is cached. This returns either the +// currently loaded value, or the current stale if the latest load errored or +// the value has expired while its stale is still valid, or the load error if +// there is no stale. If nothing is cached, or what is cached is expired with +// no valid stale, this returns Miss. func (i *Item[V]) TryGet() (v V, err error, state KeyState) { return i.c.TryGet(struct{}{}) } @@ -851,18 +1193,25 @@ func (i *Item[V]) Delete() (V, error, KeyState) { // Expire sets the item to expire immediately, meaning the next call to Get // will be a miss. If stale values are enabled, the next Get will trigger the // miss function but still allow the now-stale value to be returned. +// +// Expire only affects an item that has finished loading. Calling Expire while +// the miss function is still running is a no-op; the in-flight load will +// complete with its normal TTL and is not canceled or shortened. Expiring an +// already-expired item is also a no-op: the expiry is never moved forward. func (i *Item[V]) Expire() { i.c.Expire(struct{}{}) } // Set sets the value for the item. If the item is currently loading via Get, -// the load is canceled and Get returns the value from Set. +// the load's result is discarded (the miss function itself is not +// interrupted) and Get returns the value from Set. func (i *Item[V]) Set(v V) { i.c.Set(struct{}{}, v) } -// Swap sets the value for the item. If the item is currently loading via Get, -// the load is canceled and Get returns the value from Set. This returns the +// Swap sets the value for the item. If the item is currently loading via +// Get, the load's result is discarded (the miss function itself is not +// interrupted) and Get returns the value from Swap. This returns the // previously stored value, or the previous stale if the load errored, or the // previous error if there is no stale. If nothing is cached, this returns // Miss. @@ -871,22 +1220,45 @@ func (i *Item[V]) Swap(v V) (old V, oldErr error, oldState KeyState) { } // CompareAndSwap swaps the old and new values if the value has finished -// loading and the value is equal to old. The type V must be comparable. +// loading without an error, is not expired, and is equal to old. The type V +// must be comparable. See Cache.CompareAndSwap for the stale-handling and +// best-effort expiry caveats. func (i *Item[V]) CompareAndSwap(old, new V) (swapped bool) { return i.c.CompareAndSwap(struct{}{}, old, new) } -// CompareAndDelete deletes the item if the value has finished loading and the -// value is equal to old. The type V must be comparable. +// CompareAndDelete deletes the item if the value has finished loading +// without an error, is not expired, and is equal to old. The type V must be +// comparable. See Cache.CompareAndDelete for the stale-handling and +// best-effort expiry caveats. func (i *Item[V]) CompareAndDelete(old V) (deleted bool) { return i.c.CompareAndDelete(struct{}{}, old) } -// Clear deletes the cached item, resetting the item to an empty state. +// Clear deletes the cached item, resetting the item to an empty state. See +// Cache.Clear for the in-flight-load caveat. func (i *Item[V]) Clear() { i.c.Clear() } +// Clean deletes the item if it is expired. The item is expired if MaxAge is +// used and the item is older than the max age, or if you manually expired +// it. The item is removed only once it is invisible to every read: with +// MaxStaleAge used and not negative, its stale window runs until +// MaxAge + MaxStaleAge — anchored at the birth if the item was born expired +// (MaxAge(0) or MaxErrorAge(0)) — and it is never removed while the window +// is still open. An expired error is removed as soon as no valid stale +// remains. If MaxStaleAge is negative, Clean returns immediately. +func (i *Item[V]) Clean() { + i.c.Clean() +} + +// StopAutoClean stops the auto clean goroutine that is begun with the +// AutoCleanInterval option. +func (i *Item[V]) StopAutoClean() { + i.c.StopAutoClean() +} + ///////// // SET // ///////// @@ -899,24 +1271,21 @@ type Set[K comparable] struct { } // NewSet returns a new Set, with the optional overrides configuring cache -// semantics. If you do not need to configure an set at all, the zero value set +// semantics. If you do not need to configure a set at all, the zero value set // is valid and usable. func NewSet[K comparable](opts ...Opt) *Set[K] { - var c cfg - for _, opt := range opts { - opt.apply(&c) - } - return &Set[K]{ - c: Cache[K, struct{}]{ - cfg: c, - }, - } + s := new(Set[K]) + initCache(&s.c, opts...) + return s } // Get ensures the key is cached, running the miss function in a goroutine if -// the key is not yet cached. If stale keys are enabled, the currently cached -// key has an error, and there is a stale key, this returns with no error and a -// Stale key state. +// the key is not yet cached, the cached key has expired, or the cached key +// holds an unexpired error with a valid stale to serve. If stale keys are +// enabled and an unexpired stale exists — the currently cached key has an +// error, or it expired with its stale window still open — this returns with +// no error and a Stale key state. See Cache.Get for the miss function's +// panic and re-entrancy caveats. func (s *Set[K]) Get(k K, miss func() error) (err error, state KeyState) { _, err, state = s.c.Get(k, func() (struct{}, error) { return struct{}{}, miss() @@ -924,10 +1293,11 @@ func (s *Set[K]) Get(k K, miss func() error) (err error, state KeyState) { return err, state } -// TryGet any error for the given key if it is cached. This returns either the -// currently stored nil error, or if the current store has an error, the stale -// nil error if present, otherwise the current error. If nothing is cached, or -// what is cached is expired, this returns Miss. +// TryGet returns the cached state for the given key: nil if the key loaded +// successfully, a stale nil if a valid stale exists while the latest load +// errored or the key expired, or the load error itself. If nothing is +// cached, or what is cached is expired with no valid stale, this returns +// Miss. func (s *Set[K]) TryGet(k K) (err error, state KeyState) { _, err, state = s.c.TryGet(k) return err, state @@ -943,38 +1313,51 @@ func (s *Set[K]) Delete(k K) (error, KeyState) { // Expire sets the key to expire immediately, meaning the next call to Get will // be a miss. If stale keys are enabled, the next Get will trigger the miss // function but still allow a now-stale nil error to be returned. +// +// Expire only affects a key whose load has finished. Calling Expire while the +// miss function is still running is a no-op; the in-flight load will complete +// with its normal TTL and is not canceled or shortened. Expiring an +// already-expired key is also a no-op: the expiry is never moved forward. func (s *Set[K]) Expire(k K) { s.c.Expire(k) } // Range calls fn for every cached key. If fn returns false, iteration stops. +// See Cache.Range for the consistency caveats: no snapshot is taken, and no +// key is visited more than once. func (s *Set[K]) Range(fn func(K, error) bool) { s.c.Range(func(k K, _ struct{}, err error) bool { return fn(k, err) }) } -// Clean deletes all expired keys from the cache. A key is expired if MaxAge is -// used and the key is older than the max age, or if you manually expired a -// key. If MaxStaleAge is used and not -1, the entry must be older than MaxAge -// + MaxStaleAge. If MaxStaleAge is -1, Clean returns immediately. +// Clean deletes all expired keys from the cache. A key is expired if MaxAge +// is used and the key is older than the max age, or if you manually expired +// a key. An entry is removed only once it is invisible to every read: with +// MaxStaleAge used and not negative, its stale window runs until +// MaxAge + MaxStaleAge — anchored at the birth for entries born expired +// (MaxAge(0) or MaxErrorAge(0)) — and an entry whose window is still open +// is never removed. Expired errors are removed as soon as no valid stale +// remains. If MaxStaleAge is negative, Clean returns immediately. func (s *Set[K]) Clean() { s.c.Clean() } -// Set sets a key to exist. If the key is currently loading via Get, -// the load is canceled and Get returns nil. +// Set sets a key to exist. If the key is currently loading via Get, the +// load's result is discarded (the miss function itself is not interrupted) +// and Get returns nil. func (s *Set[K]) Set(k K) { s.c.Set(k, struct{}{}) } -// Clear deletes all keys from the set, resetting it to an empty state. +// Clear deletes all keys from the set, resetting it to an empty state. See +// Cache.Clear for the in-flight-load caveat. func (s *Set[K]) Clear() { s.c.Clear() } -// StopAutoClean stops the auto clean goroutine that is began with the -// AutoClean option. +// StopAutoClean stops the auto clean goroutine that is begun with the +// AutoCleanInterval option. func (s *Set[K]) StopAutoClean() { s.c.StopAutoClean() } diff --git a/cache/cache_test.go b/cache/cache_test.go index d170261..c573c22 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -2,6 +2,8 @@ package cache import ( "errors" + "math" + "runtime" "sync" "sync/atomic" "testing" @@ -58,7 +60,7 @@ func TestCollapsedGet(t *testing.T) { calls int64 ps = make(chan *int) ) - for i := 0; i < niter; i++ { + for range niter { go func() { p, _, _ := c.Get("foo", func() (*int, error) { if atomic.AddInt64(&calls, 1) != 1 { @@ -70,7 +72,7 @@ func TestCollapsedGet(t *testing.T) { ps <- p }() } - for i := 0; i < niter; i++ { + for range niter { p := <-ps if p != r { t.Error("pointer mismatch") @@ -118,7 +120,7 @@ func TestSetExpire(t *testing.T) { c.Delete(k) c.Set(k, 10) - for i := 0; i < 2; i++ { + for range 2 { var wg sync.WaitGroup wg.Add(10) go func() { defer wg.Done(); c.Range(func(string, int, error) bool { return false }) }() @@ -282,7 +284,7 @@ func TestExpires(t *testing.T) { gch := make(chan got[string], 10) ch := make(chan struct{}) - for i := 0; i < 10; i++ { + for range 10 { go func() { v, err, s := c.Get("foo", func() (string, error) { <-ch @@ -296,7 +298,7 @@ func TestExpires(t *testing.T) { // long, so not all Get's could be stacked at once. time.Sleep(100 * time.Millisecond) close(ch) - for i := 0; i < 10; i++ { + for range 10 { g := <-gch if g.s == Miss { vcheck(t, g, got[string]{"bar", nil, Miss}) @@ -365,22 +367,26 @@ func TestExpires(t *testing.T) { } func TestMaxIdleAge(t *testing.T) { + // Scale all time windows generously — these tests are timing-based and + // fail under heavy -race/stress load if the sleep/access margin is thin. + // MaxIdleAge alone: entry stays alive while accessed within the window, // expires after inactivity. t.Run("alone", func(t *testing.T) { - c := New[string, string](MaxIdleAge(50 * time.Millisecond)) + const idle = 200 * time.Millisecond + c := New[string, string](MaxIdleAge(idle)) c.Get("foo", func() (string, error) { return "bar", nil }) - time.Sleep(10 * time.Millisecond) + time.Sleep(40 * time.Millisecond) // Access within the idle window — should still be a hit and extend. - for i := 0; i < 5; i++ { + for range 5 { v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) - time.Sleep(30 * time.Millisecond) // each access resets the 50ms idle + time.Sleep(40 * time.Millisecond) // ≪ idle, plenty of margin } - // Now stop accessing. After 50ms of inactivity, it should expire. - time.Sleep(60 * time.Millisecond) + // Now stop accessing. After idle of inactivity, it should expire. + time.Sleep(idle + 50*time.Millisecond) v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) @@ -388,52 +394,561 @@ func TestMaxIdleAge(t *testing.T) { // MaxIdleAge + MaxAge: initial expiry uses MaxAge, subsequent accesses // extend using MaxIdleAge. t.Run("with_max_age", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + const ttl = 200 * time.Millisecond + c := New[string, string](MaxAge(ttl), MaxIdleAge(ttl)) c.Get("foo", func() (string, error) { return "bar", nil }) // Keep alive well past the original MaxAge. - for i := 0; i < 5; i++ { - time.Sleep(30 * time.Millisecond) + for range 5 { + time.Sleep(40 * time.Millisecond) v, err, s := c.Get("foo", func() (string, error) { return "baz", nil }) vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Hit}) } // Stop accessing, should expire. - time.Sleep(60 * time.Millisecond) + time.Sleep(ttl + 50*time.Millisecond) v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) // MaxIdleAge doesn't extend errors. t.Run("no_extend_errors", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + const ttl = 200 * time.Millisecond + c := New[string, string](MaxAge(ttl), MaxIdleAge(ttl)) c.Get("foo", func() (string, error) { return "", errors.New("err") }) - time.Sleep(10 * time.Millisecond) + time.Sleep(40 * time.Millisecond) // Access the errored entry — it should not be extended. v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", errors.New("err"), Hit}) // Wait past expiry — should be gone. - time.Sleep(50 * time.Millisecond) + time.Sleep(ttl + 50*time.Millisecond) + v, err, s = c.TryGet("foo") + vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) + }) + + // MaxIdleAge must not resurrect entries that finalize already expired: + // with MaxAge(0), caching is disabled entirely and the Get that drives + // (or piggybacks on) the load gets a courtesy value, not an + // idle-extending access. Regression test: the extension previously + // fired on the waited-expired path and revived the value for the idle + // window. + t.Run("no_resurrect_expired", func(t *testing.T) { + c := New[string, string](MaxAge(0), MaxIdleAge(time.Hour)) + v, err, s := c.Get("foo", func() (string, error) { return "bar", nil }) + vcheck(t, got[string]{v, err, s}, got[string]{"bar", nil, Miss}) v, err, s = c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) // Range doesn't extend. t.Run("range_no_extend", func(t *testing.T) { - c := New[string, string](MaxAge(50*time.Millisecond), MaxIdleAge(50*time.Millisecond)) + const ttl = 200 * time.Millisecond + c := New[string, string](MaxAge(ttl), MaxIdleAge(ttl)) c.Get("foo", func() (string, error) { return "bar", nil }) // Range over entries repeatedly — should NOT extend expiry. - for i := 0; i < 3; i++ { - time.Sleep(20 * time.Millisecond) + for range 3 { + time.Sleep(40 * time.Millisecond) c.Range(func(string, string, error) bool { return true }) } - // Should be expired by now (>50ms since creation, range didn't extend). - time.Sleep(20 * time.Millisecond) + // Should be expired by now (>ttl since creation, range didn't extend). + time.Sleep(ttl + 50*time.Millisecond) v, err, s := c.TryGet("foo") vcheck(t, got[string]{v, err, s}, got[string]{"", nil, Miss}) }) } + +// TestMaxErrorAge_DefaultsToIdleInitialTTL pins the error-TTL default when +// only MaxIdleAge is set (see MaxErrorAge's doc): the idle age is every +// entry's initial TTL, errors included, so a load error is cached for the +// idle age — not forever — and the next Get after it lapses retries. The +// pre-expiry probe asserts the error is the cached result within the +// window; it cannot mask a (regressed) idle extension of the errored entry, +// because it runs immediately after the load, so even an extension anchored +// at that access would lapse before the post-window probe. +func TestMaxErrorAge_DefaultsToIdleInitialTTL(t *testing.T) { + const idle = 200 * time.Millisecond + c := New[string, int](MaxIdleAge(idle)) + boom := errors.New("boom") + c.Get("k", func() (int, error) { return 0, boom }) + + if _, err, s := c.TryGet("k"); err == nil || !s.IsHit() { + t.Fatalf("TryGet within the idle window: err=%v s=%v, want the cached error as a Hit", err, s) + } + + time.Sleep(2*idle + 50*time.Millisecond) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the idle age: %v, want Miss (the error is not cached forever)", s) + } + + var retried bool + v, err, s := c.Get("k", func() (int, error) { retried = true; return 7, nil }) + if !retried || v != 7 || err != nil || s != Miss { + t.Fatalf("Get after the error lapsed: retried=%v (%d, %v, %v), want a fresh (7, nil, Miss) load", retried, v, err, s) + } +} + +// TestHugeAges verifies expiry arithmetic saturates rather than wrapping: +// absurd-but-valid Durations must mean "effectively forever", not "born +// expired" (or, for Clean, "evict everything"). +func TestHugeAges(t *testing.T) { + huge := time.Duration(math.MaxInt64) + + // MaxAge: now + ttl must not wrap negative. + { + c := New[string, int](MaxAge(huge)) + c.Set("k", 1) + if v, _, s := c.TryGet("k"); !s.IsHit() || v != 1 { + t.Fatalf("TryGet under huge MaxAge: v=%d s=%v, want 1 Hit", v, s) + } + } + + // MaxStaleAge: Clean's expires + staleAge must not wrap negative, and + // the stale's own expiry must not be born wrapped. + { + c := New[string, int](MaxAge(time.Nanosecond), MaxStaleAge(huge)) + c.Set("k", 1) + time.Sleep(time.Millisecond) // main entry expired, stale effectively forever + c.Clean() // must not evict + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet after Clean under huge MaxStaleAge: v=%d s=%v, want 1 Stale", v, s) + } + } + + // MaxIdleAge: the extension n + idle must not wrap negative. + { + c := New[string, int](MaxIdleAge(huge)) + c.Set("k", 1) + c.TryGet("k") // a Hit extends; the extension must saturate + if v, _, s := c.TryGet("k"); !s.IsHit() || v != 1 { + t.Fatalf("TryGet after huge idle extension: v=%d s=%v, want 1 Hit", v, s) + } + } +} + +// TestClean_RespectsValidStales verifies Clean never removes an entry whose +// stale window is still open. Born-expired entries (MaxAge(0) stores a +// negative, birth-carrying word) are the regression case: their +// MaxAge+MaxStaleAge grace used to anchor at the epoch instead of at the +// entry, so Clean evicted stales that TryGet was still serving. +func TestClean_RespectsValidStales(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(50*time.Millisecond)) + c.Set("k", 1) + + c.Clean() // stale still valid: must not evict + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet after early Clean: v=%d s=%v, want 1 Stale", v, s) + } + + time.Sleep(60 * time.Millisecond) // stale now expired + c.Clean() // must evict + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after late Clean: s=%v, want Miss", s) + } +} + +// trieEntries counts every entry wired into the trie, including tombstoned +// shells and entries that are invisible to Range and TryGet. +func trieEntries[K comparable, V any](c *Cache[K, V]) (n int) { + c.t.walk(func(*ent[K, V]) bool { n++; return true }) + return n +} + +// TestClean_ReclaimsBornExpired verifies Clean's handling of born-expired +// entries (MaxAge(0) / MaxErrorAge(0), whose negative expiry word carries +// the negated birth): an entry is removed exactly when no read can see it +// anymore (i.e. as soon as TryGet reports Miss) — immediately for errored +// stale-less entries, and exactly when the birth-anchored stale window +// closes for values — regardless of process uptime. +// +// Regression test: born-expired entries used to store a bare -1 sentinel, +// and the grace window was computed as satAdd(-1, maxStaleAge), anchoring +// it at the process epoch — born-expired stale-less entries (error churn +// under MaxErrorAge(0)) were unreclaimable until process uptime exceeded +// MaxStaleAge, and forever for huge stale ages. +func TestClean_ReclaimsBornExpired(t *testing.T) { + t.Run("errored_no_stale", func(t *testing.T) { + c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Hour)) + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: errored entry should be invisible, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d dead invisible born-expired entries in the trie", n) + } + }) + + t.Run("huge_stale_age", func(t *testing.T) { + c := New[string, int](MaxErrorAge(0), MaxStaleAge(time.Duration(math.MaxInt64))) + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: errored entry should be invisible, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d dead entries (huge MaxStaleAge made them permanent)", n) + } + }) + + t.Run("keeps_valid_stale", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) // born expired; stale window anchored at the store + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + c.Clean() + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("Clean evicted a born-expired entry with a valid stale: TryGet=(%d,%v)", v, s) + } + if n := trieEntries(c); n != 1 { + t.Fatalf("trie entries = %d, want 1", n) + } + }) + + t.Run("reclaims_dead_stale", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(5*time.Millisecond)) + c.Set("k", 1) + time.Sleep(10 * time.Millisecond) // stale window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: want Miss, got %v", s) + } + c.Clean() + if n := trieEntries(c); n != 0 { + t.Fatalf("Clean left %d entries whose stale window had passed", n) + } + }) +} + +// TestStale_BornExpiredAnchorsAtBirth verifies that a born-expired value's +// (MaxAge(0)) stale window is anchored at the value's birth — carried by +// the negated-birth expiry word — for every read and refresh. Once the +// window passes, the value is gone for good: a later Get drives a fresh +// load rather than serving the dead value. +// +// Regression test: born-expired values used to store a bare -1 sentinel, +// destroying the birth; a Get-driven refresh arbitrarily later re-anchored +// the dead value's stale at the refresh (newStale's expires<=0 fallback), +// serving a value of unbounded age as a fresh Stale right after TryGet +// certified Miss. +func TestStale_BornExpiredAnchorsAtBirth(t *testing.T) { + const age = 50 * time.Millisecond + c := New[string, int](MaxAge(0), MaxStaleAge(age)) + c.Set("k", 1) + + // Within the window, the value is servable as a stale. + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet within the window: (%d, %v), want (1, Stale)", v, s) + } + + time.Sleep(2 * age) // window passed; the value is certifiably dead + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the window: %v, want Miss", s) + } + v, _, s := c.Get("k", func() (int, error) { return 2, nil }) + if v != 2 || s != Miss { + t.Fatalf("Get after the window: (%d, %v), want (2, Miss): the dead value must not be re-anchored at the refresh", v, s) + } +} + +// TestStale_GetLoadedValueServesSelfStale verifies the stale window applies +// uniformly regardless of how the value was written: a Get-loaded value, +// once expired, is served as Stale by TryGet for MaxStaleAge, exactly like +// a Set-loaded value. +// +// Regression test: only Set/Swap-created loadings used to carry a stale +// snapshot of themselves; Get-loaded values had none (their stale field is +// the previous generation's refresh companion), so TryGet reported Miss +// the instant they expired while a Get-driven refresh of the same state +// served the Stale. +func TestStale_GetLoadedValueServesSelfStale(t *testing.T) { + const ttl = 20 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(time.Hour)) + c.Get("k", func() (int, error) { return 1, nil }) + time.Sleep(2 * ttl) + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet on an expired Get-loaded value: (%d, %v), want (1, Stale)", v, s) + } +} + +// TestStale_ServesLatestGeneration verifies that once a refreshed value +// itself expires, every read serves that value as the stale — not the +// generation before it. +// +// Regression test: a Get-created loading's stale field is the previous +// generation's snapshot; TryGet used to serve it (v1) after the refreshed +// value (v2) expired, while a Get-driven refresh of the same state +// snapshotted and served v2. +func TestStale_ServesLatestGeneration(t *testing.T) { + const ttl = 30 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(time.Hour)) + c.Set("k", 1) + time.Sleep(ttl + 10*time.Millisecond) // v1 expired; within its stale window + + // Drive a refresh to v2; the driving Get returns the v1 stale while + // its (held-open) miss runs. + release := make(chan struct{}) + if v, _, s := c.Get("k", func() (int, error) { <-release; return 2, nil }); s != Stale || v != 1 { + t.Fatalf("refreshing Get: (%d, %v), want (1, Stale)", v, s) + } + close(release) + time.Sleep(ttl + 20*time.Millisecond) // v2 finalized, then expired + + if v, _, s := c.TryGet("k"); s != Stale || v != 2 { + t.Fatalf("TryGet after v2 expired: (%d, %v), want (2, Stale) — the freshest dead value", v, s) + } +} + +// TestMaxIdleAge_StaleWindowFollowsExtensions verifies that the stale +// window of an idle-extended entry anchors at the entry's actual death +// (its last extension lapsing), not at its initial expiry: derived from +// the expiry word, the window moves with every extension. +// +// Regression test: the self-stale used to be a snapshot frozen at +// creation, anchored at the initial expiry. Extensions moved only the +// expiry word, so an entry kept hot past initialExpiry+MaxStaleAge idled +// out with its stale window already closed: TryGet reported Miss the +// instant the entry expired while Get served the Stale. +func TestMaxIdleAge_StaleWindowFollowsExtensions(t *testing.T) { + const ( + ttl = 200 * time.Millisecond + age = 400 * time.Millisecond + ) + c := New[string, int](MaxAge(ttl), MaxIdleAge(ttl), MaxStaleAge(age)) + c.Set("k", 1) + + // Keep the entry hot well past the initial expiry + age (600ms after + // the store), so a creation-frozen window would be long dead. + for range 8 { // ~640ms of extensions + time.Sleep(80 * time.Millisecond) + if _, _, s := c.TryGet("k"); !s.IsHit() { + t.Fatal("entry should still be live while being extended") + } + } + time.Sleep(ttl + 100*time.Millisecond) // idle out: died ~100ms ago + + // ~100ms into the 400ms post-death window: still a Stale. + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet shortly after an idle-extended entry expired: (%d, %v), want (1, Stale)", v, s) + } + time.Sleep(age) // window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the stale window: %v, want Miss", s) + } +} + +// TestExpire_DoesNotResurrectDeadStale verifies that Expire on an entry +// whose value and stale window are both already dead is a no-op: the next +// Get drives a fresh load rather than serving the dead value. +// +// Regression test: Expire used to blindly store now()-1, moving an +// already-expired word forward; the next Get-driven refresh anchored a +// fresh stale window at the Expire and served the dead value as Stale for +// up to another MaxStaleAge. +func TestExpire_DoesNotResurrectDeadStale(t *testing.T) { + const ttl, age = 5 * time.Millisecond, 5 * time.Millisecond + c := New[string, int](MaxAge(ttl), MaxStaleAge(age)) + c.Set("k", 1) + time.Sleep(30 * time.Millisecond) // value expired AND stale window passed + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: want Miss, got %v", s) + } + c.Expire("k") // invalidating a dead key must not revive anything + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get after expiring a dead entry: (%d, %v), want (2, Miss)", v, s) + } +} + +// TestExpire_StaleWindowAnchorsAtExpire verifies the uniform stale-window +// anchor for manually expired values: the window opens at the Expire and +// closes MaxStaleAge later, for TryGet and Get-driven refreshes alike. +// +// Regression test: TryGet used to serve a frozen creation-time snapshot +// whose window was anchored at the original expiry (an hour out here), +// outliving the documented Expire-anchored window. +func TestExpire_StaleWindowAnchorsAtExpire(t *testing.T) { + const age = 50 * time.Millisecond + c := New[string, int](MaxAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + c.Expire("k") + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet just after Expire: (%d, %v), want (1, Stale)", v, s) + } + time.Sleep(2 * age) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet after the post-Expire window: %v, want Miss", s) + } +} + +// TestExpire_MidWindowDoesNotReanchor verifies Expire is a no-op on an +// entry that is expired but still inside its stale window: the expiry word +// (the window's anchor) must be byte-identical after the call. +// TestExpire_DoesNotResurrectDeadStale covers the window-already-closed +// case; here a forward re-anchor would extend a dead value's servable life +// mid-window. +func TestExpire_MidWindowDoesNotReanchor(t *testing.T) { + t.Run("ttl_expired", func(t *testing.T) { + c := New[string, int](MaxAge(20*time.Millisecond), MaxStaleAge(time.Hour)) + c.Set("k", 1) + time.Sleep(40 * time.Millisecond) // expired; window open for ~1h + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + c.Expire("k") + if w1 := l.expires.Load(); w1 != w0 { + t.Fatalf("Expire moved an already-expired word: %d -> %d (re-anchors the stale window)", w0, w1) + } + }) + t.Run("born_expired", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) // word carries the negated birth + if _, _, s := c.TryGet("k"); s != Stale { + t.Fatalf("precondition: want Stale, got %v", s) + } + l := c.t.loadEntry("k").p.Load() + w0 := l.expires.Load() + if w0 >= 0 { + t.Fatalf("precondition: want a negated-birth word, got %d", w0) + } + c.Expire("k") + if w1 := l.expires.Load(); w1 != w0 { + t.Fatalf("Expire destroyed the birth anchor: %d -> %d", w0, w1) + } + }) +} + +// TestClean_KeepsExpiredErrorWithValidCompanion pins Clean's gate for +// errored entries: an expired error whose refresh companion (the previous +// generation) is still inside its window is visible to reads — TryGet +// serves the companion — so Clean must not reclaim it. +func TestClean_KeepsExpiredErrorWithValidCompanion(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour), MaxErrorAge(20*time.Millisecond)) + c.Set("k", 1) + c.Expire("k") + // Drive a refresh that errors; the new loading carries companion=1. + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + e := c.t.loadEntry("k") + for l := e.p.Load(); l == nil || !l.finalized(); l = e.p.Load() { + runtime.Gosched() + } + time.Sleep(40 * time.Millisecond) // error now expired; companion valid ~1h + + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("precondition: TryGet=(%d,%v), want (1, Stale) via the companion", v, s) + } + c.Clean() + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("Clean reclaimed an entry whose companion was still servable: TryGet=(%d,%v)", v, s) + } + if n := trieEntries(c); n != 1 { + t.Fatalf("trie entries = %d, want 1", n) + } +} + +// TestStale_WindowBoundaries drives the exact-nanosecond boundaries of the +// derived stale window through loadingTryGet with hand-picked clocks (the +// n64 parameter pins the clock deterministically, exactly as Range and +// Delete do) and checks the window definitions agree bit-for-bit across +// the read path (staleOpen), the refresh companion (newStale), and +// stale.expired: the window is [expiry, expiry+age), half-open at every +// site, for positive, negated-birth, and claimed words. +func TestStale_WindowBoundaries(t *testing.T) { + const age = 100 * time.Millisecond + a := int64(age) + c := New[string, int](MaxAge(time.Hour), MaxStaleAge(age)) + c.Set("k", 1) + l := c.t.loadEntry("k").p.Load() + + check := func(n int64, wantS KeyState, wantV int, what string) { + t.Helper() + v, _, s := loadingTryGet(l, n, 0, age) + if s != wantS || (wantS != Miss && v != wantV) { + t.Fatalf("%s: loadingTryGet=(%d,%v), want (%d,%v)", what, v, s, wantV, wantS) + } + } + + // Positive word: expiry E, window [E, E+age). + E := now() - 1 + l.expires.Store(E) + check(E-1, Hit, 1, "one ns before expiry") + check(E, Stale, 1, "the expiry instant itself") + check(E+a-1, Stale, 1, "last open ns of the window") + check(E+a, Miss, 0, "the window-close instant") + + if st := newStale(1, E, age); st.expires != E+a { + t.Fatalf("companion window end %d != read-path window end %d", st.expires, E+a) + } else if st.expired(E+a-1) || !st.expired(E+a) { + t.Fatal("companion boundary disagrees with the read-path boundary") + } + if !staleOpen(E, E+a-1, age) || staleOpen(E, E+a, age) { + t.Fatal("staleOpen boundary inconsistent with itself") + } + + // Negated-birth word: anchor B, window [B, B+age). + B := now() + l.expires.Store(-B) + check(B+a-1, Stale, 1, "born-expired: last open ns") + check(B+a, Miss, 0, "born-expired: window-close instant") + if st := newStale(1, -B, age); st.expires != B+a { + t.Fatalf("born-expired companion end %d, want %d", st.expires, B+a) + } + + // Claimed word: no window, ever; and no companion may be derived. + l.expires.Store(expiredClaimed) + check(B, Miss, 0, "claimed word") + if staleOpen(expiredClaimed, 1, age) { + t.Fatal("staleOpen derived a window from a claimed word") + } + if st := entMaybeNewStale(c.t.loadEntry("k"), age); st != nil { + t.Fatalf("entMaybeNewStale snapshotted a claimed loading: %+v", st) + } +} + +// TestCompareAndSwapAgreesWithTryGet verifies that CompareAndSwap and +// CompareAndDelete only match live values: expired entries and errored +// entries are "not there" per TryGet, so comparing against them must fail. +func TestCompareAndSwapAgreesWithTryGet(t *testing.T) { + t.Run("expired", func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + c.Expire("k") + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatal("expired entry should TryGet Miss") + } + if c.CompareAndSwap("k", 1, 2) { + t.Error("CompareAndSwap matched an expired value") + } + if c.CompareAndDelete("k", 1) { + t.Error("CompareAndDelete matched an expired value") + } + }) + + t.Run("errored", func(t *testing.T) { + c := New[string, int]() + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + // The errored loading's v is the zero int; CAS against 0 must not + // treat that as a cached value. + if c.CompareAndSwap("k", 0, 2) { + t.Error("CompareAndSwap matched an errored entry's placeholder value") + } + if c.CompareAndDelete("k", 0) { + t.Error("CompareAndDelete matched an errored entry's placeholder value") + } + }) + + t.Run("live", func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + if !c.CompareAndSwap("k", 1, 2) { + t.Error("CompareAndSwap failed on a live matching value") + } + if !c.CompareAndDelete("k", 2) { + t.Error("CompareAndDelete failed on a live matching value") + } + }) +} diff --git a/cache/concurrency_test.go b/cache/concurrency_test.go new file mode 100644 index 0000000..e482056 --- /dev/null +++ b/cache/concurrency_test.go @@ -0,0 +1,1602 @@ +package cache + +import ( + "errors" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestSwap_CancelsInFlightGet verifies that Swap overrides a currently-loading +// entry: the goroutine waiting in Get sees the Swap value, and the underlying +// miss function's return value is discarded. +// +// Covers cache.go Swap defer block for !was.finalized() (the cancel path). +func TestSwap_CancelsInFlightGet(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + missDone := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Add(1) + go func() { + defer getDone.Done() + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + close(missDone) + return 99, nil + }) + getV = v + }() + + <-missStart // miss function has begun + + // Swap while the miss is still running. + _, _, oldS := c.Swap("k", 7) + if oldS != Miss { + t.Fatalf("Swap old state: got %v want Miss", oldS) + } + + // Let the miss function finish. Its value MUST be discarded. + close(missRelease) + <-missDone // ensure the miss goroutine actually ran to completion + getDone.Wait() // ensure the Get goroutine returned + + if getV != 7 { + t.Fatalf("Get value = %d, want 7 (Swap should cancel miss)", getV) + } + + // Verify the cache holds Swap's value, not the miss's. + v, _, s := c.TryGet("k") + if v != 7 || !s.IsHit() { + t.Fatalf("TryGet after Swap: v=%d s=%v, want 7 Hit", v, s) + } +} + +// TestSwap_OverExpiredFinalizedWithStale exercises the defer branch that +// reads the prior entry's stale when the prior entry has expired but a valid +// stale is still in its window. +func TestSwap_OverExpiredFinalizedWithStale(t *testing.T) { + c := New[string, int]( + MaxAge(5*time.Millisecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(15 * time.Millisecond) + // Entry is expired but stale is still valid. + old, _, oldS := c.Swap("k", 2) + if oldS != Stale { + t.Fatalf("Swap state: got %v want Stale", oldS) + } + if old != 1 { + t.Fatalf("Swap old: got %d want 1", old) + } +} + +// TestSwap_OverErroredWithStale covers the defer branch where the prior +// entry is finalized but holds an error, and a valid stale exists. +func TestSwap_OverErroredWithStale(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour)) + c.Set("k", 1) + // Replace the value with an error result. We drive this via Get so the + // cached state becomes (err, stale=1). + c.Expire("k") + _, _, _ = c.Get("k", func() (int, error) { + return 0, errors.New("boom") + }) + // Now the entry has err set and stale=1. + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale { + t.Fatalf("Swap state: got %v want Stale", oldS) + } + if old != 1 || oldErr != nil { + t.Fatalf("Swap old: v=%d err=%v, want v=1 err=nil", old, oldErr) + } +} + +// TestDelete_DuringInFlightLoad verifies that deleting while a load is in +// flight: (a) the in-flight Get still returns the miss fn's result, and (b) +// a subsequent Get re-triggers the miss fn. +func TestDelete_DuringInFlightLoad(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Add(1) + go func() { + defer getDone.Done() + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 5, nil + }) + getV = v + }() + + <-missStart + c.Delete("k") + close(missRelease) + getDone.Wait() + + if getV != 5 { + t.Fatalf("in-flight Get got %d, want 5", getV) + } + + // Next Get must re-run the miss. + var called bool + v, _, _ := c.Get("k", func() (int, error) { + called = true + return 6, nil + }) + if !called { + t.Fatal("miss function should have run after Delete") + } + if v != 6 { + t.Fatalf("post-Delete Get got %d, want 6", v) + } +} + +// TestClear_DoesNotCacheInFlightLoads pins Clear's documented interplay with +// in-flight loads (the per-key Delete caveat applied wholesale): Clear swaps +// the trie root, so a load in flight at the Clear keeps running against the +// detached tree — its result still reaches the Get waiting on it, but is +// never cached, and a later Get misses and drives a fresh load. +func TestClear_DoesNotCacheInFlightLoads(t *testing.T) { + c := New[string, int]() + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getV int + var getDone sync.WaitGroup + getDone.Add(1) + go func() { + defer getDone.Done() + v, _, _ := c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 5, nil + }) + getV = v + }() + + <-missStart + c.Clear() + close(missRelease) + getDone.Wait() + + if getV != 5 { + t.Fatalf("in-flight Get got %d, want 5 (the load's result must still reach its waiter)", getV) + } + + // The result must not have been cached: the next Get re-runs the miss. + var called bool + v, _, _ := c.Get("k", func() (int, error) { + called = true + return 6, nil + }) + if !called { + t.Fatal("miss function should have run after Clear (the in-flight result must not be cached)") + } + if v != 6 { + t.Fatalf("post-Clear Get got %d, want 6", v) + } +} + +// TestExpire_NoOpDuringInFlightLoad verifies the documented behavior: Expire +// on an in-flight load is a no-op; the load completes with its normal TTL. +func TestExpire_NoOpDuringInFlightLoad(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + missStart := make(chan struct{}) + missRelease := make(chan struct{}) + + var getDone sync.WaitGroup + getDone.Add(1) + go func() { + defer getDone.Done() + c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 11, nil + }) + }() + + <-missStart + c.Expire("k") // must be a no-op while loading + close(missRelease) + getDone.Wait() + + v, _, s := c.TryGet("k") + if v != 11 || !s.IsHit() { + t.Fatalf("after in-flight Expire: v=%d s=%v, want 11 Hit (Expire should not have affected the load)", v, s) + } +} + +// TestCollapsing_NoCaching runs many concurrent Gets against a cache with +// MaxAge(0) (no caching). The documented use case is collapsing simultaneous +// queries for the same key. +// +// The guarantee is weaker than it looks: with del0, the loading finalizes +// expired, so the next goroutine that re-checks the entry after the leader's +// setve sees an expired entry and triggers a fresh miss. Concurrent arrivals +// still collapse in batches, but "batch" here means "whatever queued on a +// given loading's WaitGroup before it was Done'd," which is scheduler- +// dependent. +// +// We assert: (1) miss runs at least once, (2) collapsing happens (fewer miss +// calls than workers), (3) every returned value corresponds to some miss +// invocation. +func TestCollapsing_NoCaching(t *testing.T) { + c := New[string, int](MaxAge(0)) + const workers = 100 + + var missStart sync.WaitGroup + missStart.Add(1) + missRelease := make(chan struct{}) + var missCalls int32 + + var wg sync.WaitGroup + wg.Add(workers) + vals := make([]int, workers) + for i := range workers { + go func(i int) { + defer wg.Done() + v, _, _ := c.Get("k", func() (int, error) { + n := atomic.AddInt32(&missCalls, 1) + if n == 1 { + missStart.Done() + <-missRelease + } + return int(n), nil + }) + vals[i] = v + }(i) + } + + missStart.Wait() + time.Sleep(5 * time.Millisecond) // let as many callers as possible queue + close(missRelease) + wg.Wait() + + calls := atomic.LoadInt32(&missCalls) + if calls < 1 { + t.Fatalf("miss never called") + } + if calls >= workers { + t.Fatalf("no collapsing observed: miss called %d times across %d Gets", calls, workers) + } + for i, v := range vals { + if v < 1 || int32(v) > calls { + t.Fatalf("goroutine %d got value %d, outside valid miss-call range [1,%d]", i, v, calls) + } + } + + // Nothing is cached. A fresh Get runs the miss again. + var called bool + c.Get("k", func() (int, error) { + called = true + return 99, nil + }) + if !called { + t.Fatal("after collapsed Get, cache should not have retained a value") + } +} + +// TestGet_StaleRefreshCASRace pounds on the Get slow-path CAS-retry that +// replaces a finalized-expired-with-stale loading with a fresh loading +// whose stale is a snapshot of the old value. Many concurrent Gets on the +// same key force CAS losses and thus exercise the retry. +func TestGet_StaleRefreshCASRace(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + for range 200 { + c := New[string, int]( + MaxAge(time.Microsecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(time.Millisecond) // entry expired, stale valid + + const workers = 16 + var wg sync.WaitGroup + wg.Add(workers) + for range workers { + go func() { + defer wg.Done() + c.Get("k", func() (int, error) { + // Slow miss so concurrent Gets all pile into the + // slow-path stale branch before any finalizes. + time.Sleep(time.Millisecond) + return 2, nil + }) + }() + } + wg.Wait() + } +} + +// TestGet_ConcurrentStaleDuringInFlight exercises Get's slow-path branch +// where a second Get observes a stale-returning loading that is already +// in flight from the first Get. The second Get must return the stale +// without attempting to install a new loading. +func TestGet_ConcurrentStaleDuringInFlight(t *testing.T) { + c := New[string, int]( + MaxAge(time.Millisecond), + MaxStaleAge(time.Hour), + ) + c.Set("k", 1) + time.Sleep(10 * time.Millisecond) // entry now expired, stale still valid + + missRelease := make(chan struct{}) + missStart := make(chan struct{}) + var aDone sync.WaitGroup + aDone.Add(1) + go func() { + defer aDone.Done() + c.Get("k", func() (int, error) { + close(missStart) + <-missRelease + return 2, nil + }) + }() + <-missStart + + // Second Get observes the first Get's in-flight loading and its stale. + v, _, s := c.Get("k", func() (int, error) { + t.Error("second Get's miss function should not run while first Get is in flight") + return 3, nil + }) + if s != Stale || v != 1 { + t.Fatalf("second Get: v=%d s=%v, want v=1 Stale", v, s) + } + + close(missRelease) + aDone.Wait() +} + +// TestClean_SkipsInFlightLoads verifies Clean leaves pending loads alone. +// An entry whose loading hasn't finalized yet has no expiry to evaluate; +// Clean must skip it. +func TestClean_SkipsInFlightLoads(t *testing.T) { + c := New[string, int](MaxAge(time.Nanosecond)) + missRelease := make(chan struct{}) + missStarted := make(chan struct{}) + getDone := make(chan struct{}) + go func() { + defer close(getDone) + c.Get("k", func() (int, error) { + close(missStarted) + <-missRelease + return 42, nil + }) + }() + <-missStarted + // The entry exists in the trie with an in-flight loading. Clean must + // observe !l.finalized() and skip. + c.Clean() + close(missRelease) + <-getDone +} + +// TestCleanUnderConcurrency stresses Clean racing with concurrent +// Set/Get/Delete to catch races in the clean path. Correctness here is that +// the test finishes without -race complaints or panics; we also assert that +// no value ever mysteriously appears that we never Set. +func TestCleanUnderConcurrency(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]( + MaxAge(time.Millisecond), + MaxStaleAge(time.Millisecond), + ) + + var stop atomic.Bool + var wg sync.WaitGroup + + // Setters. + for w := range 4 { + wg.Add(1) + go func(w int) { + defer wg.Done() + for !stop.Load() { + for i := range 64 { + c.Set(i, i*1000+w) + } + } + }(w) + } + + // Getters. + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range 64 { + v, _, s := c.TryGet(i) + if s.IsHit() { + // Value is i*1000+worker; must satisfy the invariant. + if v%1000 > 3 || v/1000 != i { + t.Errorf("impossible value v=%d for key=%d", v, i) + return + } + } + } + } + }() + } + + // Deleters. + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range 64 { + c.Delete(i) + } + } + }() + } + + // Cleaner. + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + c.Clean() + time.Sleep(200 * time.Microsecond) + } + }() + + time.Sleep(500 * time.Millisecond) + stop.Store(true) + wg.Wait() +} + +// TestSwap_DeleteRangeChurn stresses Swap's unlocked fast path racing +// Delete's tombstone-then-unlink, with Range walking the trie throughout: +// Swap must observe tombstones and re-create keys (never resurrect a dying +// entry), and Range must tolerate entries being unlinked mid-walk. The +// interesting windows are ~nanoseconds wide; we stress from many angles for +// long enough that they land, leaving a generous budget for slow CI runners. +func TestSwap_DeleteRangeChurn(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + const readKeys = 32 + // Prime stable keys into the read map. + for i := range readKeys { + c.Set(i, i) + } + c.Range(func(int, int, error) bool { return true }) + + var wg sync.WaitGroup + var stop atomic.Bool + + // Deleters churn the primed keys, repeatedly creating the tombstone + + // unlink windows that Swap's fast path must handle. + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range readKeys { + c.Delete(i) + } + } + }() + } + // Swappers on the same primed keys exercise the unlocked fast-path CAS + // that races with the deleters. + for w := range 8 { + wg.Add(1) + go func(w int) { + defer wg.Done() + for !stop.Load() { + for i := range readKeys { + c.Swap(i, i*1000+w) + } + } + }(w) + } + // Churners add+delete NEW keys continuously so the trie's shape keeps + // changing (inserts expand slots, deletes prune empty parents) while + // Range walks it. + for w := range 4 { + wg.Add(1) + go func(w int) { + defer wg.Done() + base := readKeys + w*10000 + for !stop.Load() { + for i := range 64 { + c.Set(base+i, i) + c.Delete(base + i) + } + } + }(w) + } + // Range callers walk the trie throughout the churn. + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + c.Range(func(int, int, error) bool { return true }) + } + }() + } + + time.Sleep(2 * time.Second) + stop.Store(true) + wg.Wait() +} + +// TestTryCAS_RetryOnRace exercises the tryCAS CAS-loop retry path by running +// many concurrent CompareAndSwap calls for the same key. When one thread's CAS +// succeeds, others retry (loading the new value) and may observe a value that +// no longer matches `old`, exiting the loop. +func TestTryCAS_RetryOnRace(t *testing.T) { + c := New[string, int]() + c.Set("k", 0) + + const workers = 32 + const iters = 1000 + + var wg sync.WaitGroup + var successes int32 + wg.Add(workers) + for range workers { + go func() { + defer wg.Done() + for range iters { + for { + v, _, s := c.TryGet("k") + if !s.IsHit() { + break + } + if c.CompareAndSwap("k", v, v+1) { + atomic.AddInt32(&successes, 1) + break + } + runtime.Gosched() + } + } + }() + } + wg.Wait() + + v, _, _ := c.TryGet("k") + if int32(v) != successes { + t.Fatalf("value %d != successful CAS count %d", v, successes) + } + if successes != workers*iters { + t.Fatalf("successes %d != expected %d (every worker should eventually succeed its increments)", successes, workers*iters) + } +} + +// TestCompareAndDelete_RaceOnDelete stresses the CompareAndDelete path with +// concurrent deletes/sets to exercise both the retry loop inside tryCAS and +// the value-mismatch path after a concurrent mutation. +func TestCompareAndDelete_RaceOnDelete(t *testing.T) { + c := New[int, int]() + const workers = 16 + var wg sync.WaitGroup + wg.Add(workers * 2) + + var setDone atomic.Int64 + for w := range workers { + go func(w int) { + defer wg.Done() + for i := range 1000 { + c.Set(i%8, w) + setDone.Add(1) + } + }(w) + } + for range workers { + go func() { + defer wg.Done() + for i := range 1000 { + v, _, s := c.TryGet(i % 8) + if s.IsHit() { + c.CompareAndDelete(i%8, v) + } + } + }() + } + wg.Wait() + // No assertion on final state; the race-detector + lack of panic is the + // signal. +} + +// TestSetve_RacesWithSwap runs many iterations where a slow miss function +// races against a Swap on the same key. The purpose is to exercise the +// double-check-under-mutex branch of loading.setve (miss grabs l.mu after +// Swap has already finalized l under the same mutex in its defer block). +// +// The branch is race-dependent; we run many iterations to land it. The test +// is correctness-only (value observed by all Gets is consistent). +func TestSetve_RacesWithSwap(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 200 { + c := New[string, int]() + missRelease := make(chan struct{}) + var wg sync.WaitGroup + var gotV int32 + wg.Add(1) + go func() { + defer wg.Done() + v, _, _ := c.Get("k", func() (int, error) { + <-missRelease + return 1, nil + }) + atomic.StoreInt32(&gotV, int32(v)) + }() + // Allow the miss goroutine to enter the function, then release it + // close in time with the Swap. + runtime.Gosched() + close(missRelease) + old, _, _ := c.Swap("k", 2) + _ = old + wg.Wait() + v := atomic.LoadInt32(&gotV) + if v != 1 && v != 2 { + t.Fatalf("iter %d: got %d, want 1 or 2", iter, v) + } + } +} + +// TestRange_DeleteSetChurn races Range against a goroutine that +// continuously Deletes and re-Sets the same keys: every key cycles through +// tombstone, physical unlink, and fresh re-creation while the walk is in +// flight. Correctness is no panic and no race-detector report. +func TestRange_DeleteSetChurn(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + for i := range 128 { + c.Set(i, i) + } + + var wg sync.WaitGroup + var stop atomic.Bool + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range 128 { + c.Delete(i) + c.Set(i, i) + } + } + }() + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + c.Range(func(int, int, error) bool { return true }) + } + }() + time.Sleep(100 * time.Millisecond) + stop.Store(true) + wg.Wait() +} + +// TestClean_NoopWhenStalesArePermanent covers the MaxStaleAge<0 early return +// in Clean (the "stales kept forever" opt-in). +func TestClean_NoopWhenStalesArePermanent(t *testing.T) { + c := New[string, int](MaxAge(time.Nanosecond), MaxStaleAge(-1)) + c.Set("k", 1) + time.Sleep(time.Millisecond) // main entry is now expired, stale is forever + c.Clean() // must be a no-op + // The stale is still queryable. + _, _, s := c.TryGet("k") + if s != Stale { + t.Fatalf("after Clean with MaxStaleAge=-1: state=%v want Stale", s) + } +} + +// TestGet_RacesSwapOnExpired targets Get's slow-path Hit: the fast path +// sees an expired entry (Miss), but by the time the slow path re-checks, a +// concurrent Swap has published a fresh value for the same key. If Get +// reports a Hit it must be the Swap's value, never the expired one. Hitting +// the exact timing window requires multiple iterations. +func TestGet_RacesSwapOnExpired(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 5000 { + c := New[int, int](MaxAge(time.Hour)) + c.Set(1, 1) + c.Range(func(int, int, error) bool { return true }) + c.Expire(1) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + c.Swap(1, iter+100) + }() + go func() { + defer wg.Done() + v, _, s := c.Get(1, func() (int, error) { return -1, nil }) + if s.IsHit() && v != iter+100 { + t.Errorf("iter %d: Hit but v=%d, want %d", iter, v, iter+100) + } + }() + wg.Wait() + } +} + +// TestCompareAndSwap_RacesSet races CompareAndSwap against the first Set of +// the same key (plus a Range walk): the CAS may observe no entry, an +// in-flight entry, or the finalized value, and must succeed only in the +// last case. Race window; stress only. +func TestCompareAndSwap_RacesSet(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 5000 { + c := New[int, int]() + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + c.Set(iter, iter) + c.Range(func(int, int, error) bool { return true }) + }() + go func() { + defer wg.Done() + c.CompareAndSwap(iter, iter, iter+1) + }() + wg.Wait() + } +} + +// TestSwap_DeleteSwapRangeCycles cycles Delete-then-Swap on a fixed key set +// while Range walks: each cycle drives an entry through tombstone, unlink, +// and re-creation, so Swap's slow path repeatedly encounters entries +// mid-deletion. Race-dependent; stress only. +func TestSwap_DeleteSwapRangeCycles(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + for i := range 64 { + c.Set(i, i) + } + c.Range(func(int, int, error) bool { return true }) + + var stop atomic.Bool + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + for i := range 64 { + c.Delete(i) + c.Swap(i, i*2) + } + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + c.Range(func(int, int, error) bool { return true }) + } + }() + time.Sleep(100 * time.Millisecond) + stop.Store(true) + wg.Wait() +} + +// TestMemoryBounded churns a fixed key set in and out of the cache. We don't +// assert exact bytes; we assert that after N rounds of (Set, Delete, Range), +// the resident entry count is bounded by the number of live keys. This would +// catch a regression where Delete stopped physically unlinking entries. +func TestMemoryBounded(t *testing.T) { + c := New[int, int]() + const liveKeys = 32 + const rounds = 2000 + + for r := range rounds { + for i := range liveKeys { + c.Set(r*liveKeys+i, i) + } + // Delete the ones we just wrote except for `liveKeys` canonical keys. + for i := range liveKeys { + c.Delete(r*liveKeys + i) + } + for i := range liveKeys { + c.Set(i, i) + } + c.Range(func(int, int, error) bool { return true }) + } + + count := 0 + c.Range(func(int, int, error) bool { count++; return true }) + if count > liveKeys { + t.Fatalf("after %d rounds: %d live entries, want <= %d", rounds, count, liveKeys) + } +} + +// TestSwap_NotLostDuringConcurrentDelete verifies that a Swap racing a +// Delete of the same key is never silently dropped. The dangerous +// interleaving: Delete tombstones the value slot, Swap observes the +// tombstone, and Delete's physical unlink races Swap's installation of the +// new value. A nil value slot is terminal — Swap must re-create the key as +// a fresh entry, never write into the dying one — so if Swap returns Miss +// ("I stored into nothing"), it linearized after the Delete and the swapped +// value must be visible afterwards. +// +// Regression test: an earlier version resurrected tombstones with a bare +// CAS not serialized with deleteEntryIf's unlink, losing the swapped value +// in ~0.03% of rounds. +func TestSwap_NotLostDuringConcurrentDelete(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const rounds = 100_000 + for i := range rounds { + c := New[int, int]() + c.Set(i, 0) + + var wg sync.WaitGroup + wg.Add(2) + var swapState KeyState + go func() { + defer wg.Done() + c.Delete(i) + }() + go func() { + defer wg.Done() + _, _, swapState = c.Swap(i, 1) + }() + wg.Wait() + + if swapState == Miss { + if v, _, s := c.TryGet(i); s.IsMiss() || v != 1 { + t.Fatalf("round %d: Swap returned Miss (stored after the Delete) but TryGet=(%d,%v); the swapped value was silently dropped", i, v, s) + } + } + } +} + +// TestClean_DoesNotEvictFreshEntries runs Clean continuously against Gets +// on a cache with no MaxAge and no Deletes: nothing ever expires, so Clean +// must never remove anything and every completed Get must be a subsequent +// TryGet hit. +// +// Regression test: an earlier version created Get's entry with a nil value +// slot and CAS'd the loading in afterwards; in that window the fresh entry +// was indistinguishable from a Delete tombstone and Clean would unlink it, +// losing the cached value (and with it, request collapsing for that key). +func TestClean_DoesNotEvictFreshEntries(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int]() + + stop := make(chan struct{}) + var cleanWg sync.WaitGroup + cleanWg.Add(1) + go func() { + defer cleanWg.Done() + for { + select { + case <-stop: + return + default: + c.Clean() + runtime.Gosched() + } + } + }() + + const rounds = 200_000 + for i := range rounds { + c.Get(i, func() (int, error) { return i, nil }) + if v, _, s := c.TryGet(i); s.IsMiss() || v != i { + t.Fatalf("round %d: freshly loaded entry evicted by Clean: TryGet=(%d,%v)", i, v, s) + } + } + close(stop) + cleanWg.Wait() +} + +// TestGet_DoesNotRedriveFreshlyFinalizedValue targets the slow-path window +// between Get's load of prev and entGet's own load of the slot: if the +// in-flight load it observed finalizes live in that window, Get must return +// the Hit rather than demote the fresh value to a stale and drive a +// redundant load. Every round arranges an expired-with-stale entry, races +// Gets against the first load's release, and asserts the miss function ran +// exactly once. +func TestGet_DoesNotRedriveFreshlyFinalizedValue(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + for iter := range 2000 { + c := New[int, int](MaxAge(time.Hour), MaxStaleAge(time.Hour)) + c.Set(iter, 1) + c.Expire(iter) // expired with a valid self-stale + + // A driving Get returns the stale and runs its miss on a + // background goroutine, so completion must be synchronized on: + // missDone is closed by the (single) miss, and a redundant + // second drive panics on the double close. + release := make(chan struct{}) + missDone := make(chan struct{}) + miss := func() (int, error) { + <-release // closed below; late drivers pass straight through + close(missDone) + return 2, nil + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + c.Get(iter, miss) // drives the refresh, returns Stale(1) + }() + go func() { + defer wg.Done() + runtime.Gosched() + close(release) + }() + // Meanwhile, hammer Gets that race the finalization. None may + // drive a second load: each must piggyback the stale or return + // the fresh Hit. + for range 4 { + c.Get(iter, miss) + } + wg.Wait() + <-missDone + } +} + +// TestExpire_NotLostToIdleExtension verifies that Expire racing an +// idle-extending read sticks: once both return, the key must be expired. +// +// Regression test: idle extension previously used a plain Store on the +// expiry, which could overwrite a concurrent Expire (read loads expiry, +// Expire stores now-1, read stores now+idle). Extension now CASes against +// the expiry it observed, so an interleaved Expire wins. +func TestExpire_NotLostToIdleExtension(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + c := New[int, int](MaxIdleAge(time.Hour)) + const rounds = 100_000 + for i := range rounds { + c.Set(i, 1) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + c.TryGet(i) // a Hit extends the idle expiry + }() + go func() { + defer wg.Done() + c.Expire(i) + }() + wg.Wait() + if _, _, s := c.TryGet(i); s.IsHit() { + t.Fatalf("round %d: Expire was overwritten by a concurrent idle extension", i) + } + } +} + +// TestClean_ClaimWindowDoesNotResurrectDeadStale pins Clean's claim protocol +// against racing Gets: Clean claims an entry's expiry word before +// tombstoning, and a Get that lands between the claim and the tombstone +// snapshots that loading for a stale. The claimed word must read as "stale +// window certified closed" — never as something a stale window can be +// derived from — or the Get would serve a value past MaxAge+MaxStaleAge as +// a fresh Stale for up to another MaxStaleAge. +// +// The test performs Clean's claim by hand (same gate, same CAS) so the +// window between Clean's two CASes is held open deterministically. +// +// Regression test: Clean used to claim with -1, the born-expired sentinel, +// which newStale then re-anchored at now (born-expired words now carry +// their birth and anchor there, and a claimed word derives no window at +// all). +func TestClean_ClaimWindowDoesNotResurrectDeadStale(t *testing.T) { + c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) + c.Set("k", 1) + time.Sleep(5 * time.Millisecond) // expired AND past the stale window + + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: entry should be fully dead, got %v", s) + } + + // Clean's claim, by hand: same gate, same CAS, protocol frozen between + // Clean's two CASes. + e := c.t.loadEntry("k") + l := e.p.Load() + expires := l.expires.Load() + tn := now() + if expired := expires != 0 && expires <= tn; !expired || + (l.stale != nil && !l.stale.expired(tn)) || + (l.err == nil && staleOpen(expires, tn, c.cfg.maxStaleAge)) { + t.Fatalf("precondition: entry not Clean-eligible: expires=%d", expires) + } + if !l.expires.CompareAndSwap(expires, expiredClaimed) { + t.Fatal("claim CAS failed") + } + + // A Get in the claim window must not serve the dead value as a stale: + // the stale snapshot must come up empty, so the Get blocks on (and + // returns) the fresh load. + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get in Clean's claim window: v=%d s=%v, want 2 Miss (a Stale here resurrects a certified-dead value)", v, s) + } + if l2 := e.p.Load(); l2 == nil || l2.stale != nil { + t.Fatalf("the replacement loading must carry no stale snapshot of the claimed value, got %+v", l2) + } + + // Finish Clean's protocol: the tombstone CAS must fail (the Get + // replaced the slot) and the fresh value must survive. + if e.p.CompareAndSwap(l, nil) { + t.Fatal("Clean's tombstone CAS should have failed against the Get's replacement") + } + if v, _, s := c.TryGet("k"); v != 2 || !s.IsHit() { + t.Fatalf("after claim window: TryGet=(%d,%v), want (2,Hit)", v, s) + } +} + +// TestClean_ClaimWindowVsGetRace hammers the window between Clean's two +// CASes (the expiry-word claim and the tombstone) with racing Gets. Every +// round populates keys whose value AND stale are certifiably dead (TryGet +// returned Miss), then races a real Clean against a Get scan: any Get that +// returns Stale resurrected a certified-dead value. +// +// Probabilistic tripwire for the protocol pinned deterministically by +// TestClean_ClaimWindowDoesNotResurrectDeadStale; before Clean claimed with +// a dedicated sentinel, this fired within a few seconds under -race. +func TestClean_ClaimWindowVsGetRace(t *testing.T) { + if testing.Short() { + t.Skip("skipping race-iteration test in -short") + } + const ( + keys = 1024 + // Small enough that entries die within a round's sleep; large + // enough that a wrongly re-anchored stale stays observable for the + // Get that snapshotted it. + staleAge = 5 * time.Millisecond + ) + c := New[int, int](MaxAge(time.Nanosecond), MaxStaleAge(staleAge)) + miss := func() (int, error) { return 2, nil } + for start := time.Now(); time.Since(start) < 3*time.Second; { + for k := range keys { + c.Set(k, 1) + } + time.Sleep(staleAge + 2*time.Millisecond) + for k := range keys { + for { + if _, _, s := c.TryGet(k); s.IsMiss() { + break // certified: expired and stale dead + } + } + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + c.Clean() + }() + for k := range keys { + if v, _, s := c.Get(k, miss); s == Stale { + t.Fatalf("key %d: Get racing Clean returned a certified-dead value as a stale: v=%d", k, v) + } + } + wg.Wait() + } +} + +// TestExpire_DoesNotOverwriteCleanClaim pins Expire against Clean's claim +// window: once Clean has claimed an entry's expiry word (certifying the +// value expired and its stale window closed), a racing Expire must leave +// the claim in place — never replace it with a fresh timestamp that a Get +// in the window could re-derive a stale window from. Same hand-driven +// protocol as TestClean_ClaimWindowDoesNotResurrectDeadStale. +// +// Regression test: Expire used to blindly store now()-1 over any finalized +// word, including expiredClaimed; a Get between Clean's two CASes then +// served the certified-dead value as a fresh Stale. +func TestExpire_DoesNotOverwriteCleanClaim(t *testing.T) { + c := New[string, int](MaxAge(time.Millisecond), MaxStaleAge(time.Millisecond)) + c.Set("k", 1) + time.Sleep(5 * time.Millisecond) // expired AND past the stale window + + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("precondition: entry should be fully dead, got %v", s) + } + + e := c.t.loadEntry("k") + l := e.p.Load() + expires := l.expires.Load() + if !l.expires.CompareAndSwap(expires, expiredClaimed) { // Clean's claim + t.Fatal("claim CAS failed") + } + + c.Expire("k") // lands between Clean's two CASes; must be a no-op + + if got := l.expires.Load(); got != expiredClaimed { + t.Fatalf("Expire overwrote Clean's claim: expires=%d, want expiredClaimed", got) + } + if v, _, s := c.Get("k", func() (int, error) { return 2, nil }); v != 2 || s != Miss { + t.Fatalf("Get in the claim window: (%d, %v), want (2, Miss)", v, s) + } +} + +// TestSwap_OverFinalizedErroredWithStale is TestSwap_OverErroredWithStale +// with the race pinned the other way: the errored load has finalized by +// the time Swap displaces it. The classification must agree with TryGet: +// the valid stale, not the error, is the prior value. +// +// Regression test: the defer's finalized classification used to assign the +// stale and then fall through to overwrite it with (v, err, Hit); the +// sibling test only exercises the unfinalized branch (Swap usually beats +// the async setve), so the finalized path went unexercised. +func TestSwap_OverFinalizedErroredWithStale(t *testing.T) { + c := New[string, int](MaxStaleAge(time.Hour)) + c.Set("k", 1) + c.Expire("k") + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + + // Wait for the async setve to finalize the errored loading. + e := c.t.loadEntry("k") + for l := e.p.Load(); l == nil || !l.finalized(); l = e.p.Load() { + runtime.Gosched() + } + + if v, err, s := c.TryGet("k"); s != Stale || v != 1 || err != nil { + t.Fatalf("TryGet: (%d, %v, %v), want (1, nil, Stale)", v, err, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale || old != 1 || oldErr != nil { + t.Fatalf("Swap: (%d, %v, %v), want (1, nil, Stale) to agree with TryGet", old, oldErr, oldS) + } +} + +// TestSwap_OverFinalizedErroredNoStale covers the remaining finalized +// classification arm: with no stale to return, an unexpired cached error +// is itself the prior result, and Swap reports it as a Hit carrying the +// error — agreeing with TryGet. +func TestSwap_OverFinalizedErroredNoStale(t *testing.T) { + c := New[string, int]() // errors cache forever by default + // The driving Get waits for its load, so the error is finalized here. + c.Get("k", func() (int, error) { return 0, errors.New("boom") }) + + if _, err, s := c.TryGet("k"); err == nil || !s.IsHit() { + t.Fatalf("TryGet: err=%v s=%v, want the cached error as a Hit", err, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if old != 0 || oldErr == nil || !oldS.IsHit() { + t.Fatalf("Swap: (%d, %v, %v), want (0, boom, Hit) to agree with TryGet", old, oldErr, oldS) + } +} + +// TestDelete_ConcurrentlyFinalizedLoadReportsHit pins Delete's documented +// classification race: Delete samples the clock before tombstoning, but +// reads the loading's finalized state afterwards, so a load finalizing +// between the two is reported as the removed value (a Hit) even though no +// read ever observed it cached. This is Delete's exact code sequence with +// the window held open; the returned triple is best effort there (see +// Delete's doc), and the cache state itself is unaffected. +func TestDelete_ConcurrentlyFinalizedLoadReportsHit(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + missRelease := make(chan struct{}) + getDone := make(chan struct{}) + go func() { + defer close(getDone) + c.Get("k", func() (int, error) { <-missRelease; return 7, nil }) + }() + var e *ent[string, int] + for e = c.t.loadEntry("k"); e == nil; e = c.t.loadEntry("k") { + runtime.Gosched() + } + + // Delete's sequence (see Cache.Delete), paused mid-window: + n := now() + was := entDel(e) + c.t.deleteEntryIf("k", entDead[string, int]) + close(missRelease) // the miss finalizes the already-detached loading + <-getDone + + if v, _, s := loadingTryGet(was, n, 0, c.cfg.maxStaleAge); !s.IsHit() || v != 7 { + t.Fatalf("classifying the concurrently finalized loading: (%d, %v), want (7, Hit) per Delete's documented race", v, s) + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("the key must be gone regardless: %v, want Miss", s) + } +} + +// TestGet_CollapsedErrorWaitersRedrive pins the documented MaxErrorAge(0) +// collapsing behavior (see MaxErrorAge): Gets that collapsed onto a load +// that then errors do not share the error; each waiter re-drives its own +// load in turn, so one erroring wave of N collapsed Gets issues N loads. +func TestGet_CollapsedErrorWaitersRedrive(t *testing.T) { + c := New[string, int](MaxErrorAge(0)) + var calls atomic.Int32 + release := make(chan struct{}) + started := make(chan struct{}) + var once sync.Once + miss := func() (int, error) { + if calls.Add(1) == 1 { + once.Do(func() { close(started) }) + <-release + } + return 0, errors.New("boom") + } + const waiters = 8 + var wg sync.WaitGroup + wg.Add(waiters) + for range waiters { + go func() { defer wg.Done(); c.Get("k", miss) }() + } + <-started + time.Sleep(20 * time.Millisecond) // let the others queue on the leader + close(release) + wg.Wait() + if n := calls.Load(); n != waiters { + t.Fatalf("miss ran %d times for %d collapsed Gets; with errors uncached, every Get drives exactly one load", n, waiters) + } +} + +// armExpiresPairingHook installs expiresPairingHook such that exactly the +// first read to pass through it blocks until release is closed; all later +// reads pass straight through. The hook is cleared when the test ends. +func armExpiresPairingHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + return armExpiresPairingHookNth(t, 1) +} + +// armExpiresPairingHookNth is armExpiresPairingHook parking the nth +// (1-based) paired read instead of the first, for callers whose operation +// takes multiple paired reads before the one under test. +func armExpiresPairingHookNth(t *testing.T, nth int32) (entered, release chan struct{}) { + t.Helper() + entered = make(chan struct{}) + release = make(chan struct{}) + var calls atomic.Int32 + expiresPairingHook = func() { + if calls.Add(1) == nth { + close(entered) + <-release + } + } + t.Cleanup(func() { expiresPairingHook = nil }) + return entered, release +} + +// armCASPublishHook installs casPublishHook such that exactly the first +// tryCAS to pass through it blocks until release is closed; all later calls +// pass straight through. The hook is cleared when the test ends. +func armCASPublishHook(t *testing.T) (entered, release chan struct{}) { + t.Helper() + entered = make(chan struct{}) + release = make(chan struct{}) + var fired atomic.Bool + casPublishHook = func() { + if fired.CompareAndSwap(false, true) { + close(entered) + <-release + } + } + t.Cleanup(func() { casPublishHook = nil }) + return entered, release +} + +// TestTryCAS_RevalidatesLivenessBeforePublish pins tryCAS's pre-publish +// re-validation: a CAS caller that matched a live value and was then +// descheduled (the replacement's allocation is the widest such window in +// the call) while the value's TTL lapsed must not publish — the value it +// matched is one TryGet already certifies as Miss, and no sequential order +// explains TryGet=Miss followed by CAS=true. The residual instructions +// between the re-validation and the pointer CAS remain best effort (see +// CompareAndSwap's doc). +// +// Regression test: tryCAS used to sample liveness only before the +// allocation, so the entire alloc + deschedule window sat between the +// liveness sample and the publish. +func TestTryCAS_RevalidatesLivenessBeforePublish(t *testing.T) { + const ttl = 20 * time.Millisecond + for _, m := range []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } { + t.Run(m.name, func(t *testing.T) { + c := New[string, int](MaxAge(ttl)) + c.Set("k", 1) + + entered, release := armCASPublishHook(t) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + <-entered + + // Let the matched value's TTL lapse while the caller is parked + // between its liveness sample and its publish. + for { + if _, _, s := c.TryGet("k"); s.IsMiss() { + break + } + time.Sleep(time.Millisecond) + } + close(release) + + if <-res { + t.Fatal("published against a value that expired mid-call (TryGet had already certified Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + }) + } +} + +// TestTryCAS_RevalidationCatchesExpire extends the pre-publish re-validation +// pin to manual expiry: a CAS caller that matched a live value and was then +// parked at the replacement's allocation while a concurrent Expire killed +// the value must not publish — Expire's word write is visible to the +// re-validation's fresh liveness check, the same way a lapsed TTL is. Only +// the instructions between the re-validation and the pointer CAS remain +// best effort for Expire, exactly as for a lapsing TTL (see +// CompareAndSwap's doc). +func TestTryCAS_RevalidationCatchesExpire(t *testing.T) { + for _, m := range []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } { + t.Run(m.name, func(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + + entered, release := armCASPublishHook(t) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + <-entered + + c.Expire("k") // kills the matched value while the caller is parked + close(release) + + if <-res { + t.Fatal("published against a value a concurrent Expire had already killed (TryGet certifies Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + }) + } +} + +// TestTryCAS_RevalidationPairsWordAndClock pins HOW tryCAS's liveness +// checks read expiry: as a coherent (expiry word, clock) pair, the clock +// sampled after the word it judges (expiresNow, like every read path). A +// CAS caller descheduled inside the re-validation itself — between +// sampling the clock and loading the word, in the unpaired ordering — +// would otherwise judge the fresh word against its stale clock and +// publish against a value whose TTL had lapsed, or that a completed +// Expire had killed, while it was parked: state TryGet already certifies +// as Miss, so no sequential order explains TryGet=Miss followed by +// CAS=true followed by TryGet=Hit(new). With the paired read, any kill +// completed before the pair is taken is seen; only the instructions +// between the re-validation and the pointer CAS remain best effort (see +// CompareAndSwap's doc). +// +// The pairing hook parks the caller inside the re-validation's paired +// read — the call's second paired read; the first is tryCAS's +// pre-allocation filter — and the kill lands while it is parked. +// +// Regression test: casMatches used to evaluate now() and then load the +// word (!l.expired(now())), so a deschedule between the two let both kill +// modes through the re-validation. +func TestTryCAS_RevalidationPairsWordAndClock(t *testing.T) { + const ttl = 200 * time.Millisecond + ops := []struct { + name string + op func(c *Cache[string, int]) bool + }{ + {"CompareAndSwap", func(c *Cache[string, int]) bool { return c.CompareAndSwap("k", 1, 2) }}, + {"CompareAndDelete", func(c *Cache[string, int]) bool { return c.CompareAndDelete("k", 1) }}, + } + kills := []struct { + name string + opts []Opt + kill func(c *Cache[string, int]) + }{ + {"ttl_lapse", []Opt{MaxAge(ttl)}, func(c *Cache[string, int]) { + // Let the matched value's TTL lapse while the caller is + // parked; the certifying TryGets' own paired reads pass + // straight through the one-shot hook. + for { + if _, _, s := c.TryGet("k"); s.IsMiss() { + return + } + time.Sleep(time.Millisecond) + } + }}, + {"expire", []Opt{MaxAge(time.Hour)}, func(c *Cache[string, int]) { + c.Expire("k") + }}, + } + for _, k := range kills { + t.Run(k.name, func(t *testing.T) { + for _, m := range ops { + t.Run(m.name, func(t *testing.T) { + c := New[string, int](k.opts...) + c.Set("k", 1) + + entered, release := armExpiresPairingHookNth(t, 2) + res := make(chan bool, 1) + go func() { res <- m.op(c) }() + select { + case <-entered: + case r := <-res: + t.Fatalf("op returned %v before its re-validation (pre-filter failed; value dead too early?)", r) + } + + k.kill(c) + close(release) + + if <-res { + t.Fatal("published against a value that was killed mid-validation (TryGet had already certified Miss)") + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("state after the failed publish: %v, want Miss", s) + } + }) + } + }) + } +} + +// TestTryGet_PairedReadVsIdleExtension pins the coherence of the (expiry +// word, clock) pair a read classifies with: a reader descheduled between +// loading the word and sampling the clock, with an idle extension landing +// in that window, would otherwise classify the dead pre-extension word at +// a fresh clock and report Stale (with MaxStaleAge) or Miss (without) for +// an entry that was live at every instant of its call — and a later TryGet +// returns Hit with no intervening write, a history no sequential order +// explains. expiresNow re-confirms the word after sampling the clock, so +// the read classifies the extended (live) word instead. +// +// The hook holds the deschedule window open deterministically, like the +// hand-driven claim-window tests above. +// +// Regression test: loadingTryGet and entGet used to classify whatever word +// they loaded first at whatever clock they sampled next. +func TestTryGet_PairedReadVsIdleExtension(t *testing.T) { + const ttl = 100 * time.Millisecond + for _, tt := range []struct { + name string + opts []Opt + }{ + {"no_stale_age", []Opt{MaxAge(ttl), MaxIdleAge(time.Hour)}}, + {"with_stale_age", []Opt{MaxAge(ttl), MaxIdleAge(time.Hour), MaxStaleAge(time.Hour)}}, + } { + t.Run(tt.name, func(t *testing.T) { + c := New[string, int](tt.opts...) + c.Set("k", 1) + + entered, release := armExpiresPairingHook(t) + type res struct { + v int + s KeyState + } + r1c := make(chan res, 1) + go func() { + v, _, s := c.TryGet("k") // loads the initial expiry, parks in the hook + r1c <- res{v, s} + }() + <-entered + + // A second TryGet (the hook is one-shot) hits and extends the + // entry to ~now+1h; the entry is now live well past the + // original expiry. + if v, _, s := c.TryGet("k"); s != Hit || v != 1 { + t.Fatalf("extending TryGet: (%d, %v), want (1, Hit)", v, s) + } + // Let real time pass the original expiry while the first + // reader is still parked between its two loads. + time.Sleep(ttl + 50*time.Millisecond) + close(release) + r1 := <-r1c + + if v, _, s := c.TryGet("k"); s != Hit || v != 1 { + t.Fatalf("entry must still be live after the extension: (%d, %v)", v, s) + } + if r1.s != Hit || r1.v != 1 { + t.Fatalf("paired read returned (%d, %v) for an entry that was live throughout its call, want (1, Hit)", r1.v, r1.s) + } + }) + } +} + +// TestTryGet_PairedReadVsExpire pins the safe direction of the same window: +// otherwise the word only ever moves backward (Expire, Clean's claim), and +// a parked reader lands on one side of the race or the other — Hit per the +// pre-Expire live word (its call overlaps the Expire, so ordering the read +// first explains the history) or Miss per the expired word. Whichever side +// it lands on, no stale is conjured for an entry with no stale configured. +func TestTryGet_PairedReadVsExpire(t *testing.T) { + c := New[string, int](MaxAge(time.Hour)) + c.Set("k", 1) + + entered, release := armExpiresPairingHook(t) + type res struct { + v int + s KeyState + } + r1c := make(chan res, 1) + go func() { + v, _, s := c.TryGet("k") + r1c <- res{v, s} + }() + <-entered + + c.Expire("k") // moves the word backward while the reader is parked + close(release) + r1 := <-r1c + + if r1.s == Stale { + t.Fatalf("no stale is configured; got (%d, %v)", r1.v, r1.s) + } + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("after Expire: %v, want Miss", s) + } +} + +// TestSwap_OverBornExpired pins the born-expired classification cell of +// Swap's displaced-value report against TryGet: a displaced born-expired +// value (negated-birth expiry word) is the prior value (Stale) while its +// birth-anchored window is open, and Miss after — exactly as TryGet +// reports the same state. +func TestSwap_OverBornExpired(t *testing.T) { + t.Run("window_open", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(time.Hour)) + c.Set("k", 1) + if v, _, s := c.TryGet("k"); s != Stale || v != 1 { + t.Fatalf("TryGet: (%d, %v), want (1, Stale)", v, s) + } + old, oldErr, oldS := c.Swap("k", 2) + if oldS != Stale || old != 1 || oldErr != nil { + t.Fatalf("Swap: (%d, %v, %v), want (1, nil, Stale) to agree with TryGet", old, oldErr, oldS) + } + }) + t.Run("window_closed", func(t *testing.T) { + c := New[string, int](MaxAge(0), MaxStaleAge(30*time.Millisecond)) + c.Set("k", 1) + time.Sleep(60 * time.Millisecond) + if _, _, s := c.TryGet("k"); !s.IsMiss() { + t.Fatalf("TryGet: %v, want Miss", s) + } + if _, _, oldS := c.Swap("k", 2); oldS != Miss { + t.Fatalf("Swap: %v, want Miss to agree with TryGet", oldS) + } + }) +} diff --git a/cache/linearizability_test.go b/cache/linearizability_test.go new file mode 100644 index 0000000..493abee --- /dev/null +++ b/cache/linearizability_test.go @@ -0,0 +1,214 @@ +package cache + +import ( + "fmt" + "math/rand/v2" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// This file implements a small linearizability checker: each round runs a +// handful of concurrent map operations against a single key, records every +// operation's invocation/response order and return values, and then +// verifies that SOME linearization exists — a total order of the operations +// consistent with real time (op A precedes op B if A responded before B was +// invoked) under which sequential map semantics produce exactly the +// observed returns. If no such order exists, an atomicity bug has been +// observed and the full history is dumped. +// +// Get is deliberately excluded from histories: its singleflight contract +// makes it non-atomic by design (a concurrent Swap cancels an in-flight +// load and the Get returns the Swap's value with a Miss state), which is +// documented behavior but inexpressible as a single sequential operation. +// The cache is configured with no expiry options, so the model is exactly +// a map. + +type linKind uint8 + +const ( + linSet linKind = iota + linSwap + linDelete + linTryGet + linCAS + linCAD +) + +func (k linKind) String() string { + return [...]string{"Set", "Swap", "Delete", "TryGet", "CAS", "CAD"}[k] +} + +type linOp struct { + kind linKind + arg int // value for Set/Swap, old for CAS/CAD + arg2 int // new for CAS + + retV int + retS KeyState + retB bool + + start, end int64 // global ticket counter values around the call +} + +func (o *linOp) String() string { + switch o.kind { + case linSet: + return fmt.Sprintf("Set(%d) [%d,%d]", o.arg, o.start, o.end) + case linSwap: + return fmt.Sprintf("Swap(%d)=(%d,%v) [%d,%d]", o.arg, o.retV, o.retS, o.start, o.end) + case linDelete: + return fmt.Sprintf("Delete=(%d,%v) [%d,%d]", o.retV, o.retS, o.start, o.end) + case linTryGet: + return fmt.Sprintf("TryGet=(%d,%v) [%d,%d]", o.retV, o.retS, o.start, o.end) + case linCAS: + return fmt.Sprintf("CAS(%d,%d)=%v [%d,%d]", o.arg, o.arg2, o.retB, o.start, o.end) + case linCAD: + return fmt.Sprintf("CAD(%d)=%v [%d,%d]", o.arg, o.retB, o.start, o.end) + } + return "?" +} + +type linState struct { + present bool + val int +} + +// linApply runs op against the sequential model state, reporting the next +// state and whether the operation's recorded returns are consistent with +// running at this point in the order. +func linApply(st linState, o *linOp) (linState, bool) { + switch o.kind { + case linSet: + return linState{true, o.arg}, true + case linSwap: + if st.present { + return linState{true, o.arg}, o.retS == Hit && o.retV == st.val + } + return linState{true, o.arg}, o.retS == Miss + case linDelete: + if st.present { + return linState{}, o.retS == Hit && o.retV == st.val + } + return linState{}, o.retS == Miss + case linTryGet: + if st.present { + return st, o.retS == Hit && o.retV == st.val + } + return st, o.retS == Miss + case linCAS: + if st.present && st.val == o.arg { + return linState{true, o.arg2}, o.retB + } + return st, !o.retB + case linCAD: + if st.present && st.val == o.arg { + return linState{}, o.retB + } + return st, !o.retB + } + panic("unreachable") +} + +// linearizable reports whether some real-time-consistent total order of ops +// explains every op's returns starting from initial. +func linearizable(initial linState, ops []*linOp) bool { + used := make([]bool, len(ops)) + var rec func(st linState, depth int) bool + rec = func(st linState, depth int) bool { + if depth == len(ops) { + return true + } + next: + for i, o := range ops { + if used[i] { + continue + } + // o may go next only if no other unscheduled op wholly + // preceded it in real time. + for j, p := range ops { + if !used[j] && j != i && p.end < o.start { + continue next + } + } + st2, ok := linApply(st, o) + if !ok { + continue + } + used[i] = true + if rec(st2, depth+1) { + return true + } + used[i] = false + } + return false + } + return rec(initial, 0) +} + +func TestLinearizability(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const ( + rounds = 50_000 + opsPer = 4 + valSpace = 3 // small domain so CAS/CAD old values plausibly match + ) + c := New[int, int]() // shared across rounds so the trie grows/expands/prunes + + var ticket atomic.Int64 + for round := range rounds { + k := round + initial := linState{} + if rand.IntN(2) == 1 { + initial = linState{true, 1 + rand.IntN(valSpace)} + c.Set(k, initial.val) // sequentially primed before the round + } + + ops := make([]*linOp, opsPer) + for i := range ops { + ops[i] = &linOp{ + kind: linKind(rand.IntN(6)), + arg: 1 + rand.IntN(valSpace), + arg2: 1 + rand.IntN(valSpace), + } + } + + var wg sync.WaitGroup + wg.Add(len(ops)) + for _, o := range ops { + go func(o *linOp) { + defer wg.Done() + o.start = ticket.Add(1) + switch o.kind { + case linSet: + c.Set(k, o.arg) + case linSwap: + o.retV, _, o.retS = c.Swap(k, o.arg) + case linDelete: + o.retV, _, o.retS = c.Delete(k) + case linTryGet: + o.retV, _, o.retS = c.TryGet(k) + case linCAS: + o.retB = c.CompareAndSwap(k, o.arg, o.arg2) + case linCAD: + o.retB = c.CompareAndDelete(k, o.arg) + } + o.end = ticket.Add(1) + }(o) + } + wg.Wait() + + if !linearizable(initial, ops) { + var dump strings.Builder + for _, o := range ops { + dump.WriteString("\n\t") + dump.WriteString(o.String()) + } + t.Fatalf("round %d: no linearization explains this history (initial present=%v val=%d):%s", + round, initial.present, initial.val, dump.String()) + } + } +} diff --git a/cache/map_bench_test.go b/cache/map_bench_test.go index aeec419..4c8625f 100644 --- a/cache/map_bench_test.go +++ b/cache/map_bench_test.go @@ -513,3 +513,15 @@ func BenchmarkCompareAndDeleteMostlyMisses(b *testing.B) { }, }) } + +func BenchmarkClear(b *testing.B) { + benchMap(b, bench{ + perG: func(b *testing.B, pb *testing.PB, i int, m mapInterface[int]) { + for ; pb.Next(); i++ { + k, v := i%256, i%256 + m.Clear() + m.Store(k, v) + } + }, + }) +} diff --git a/cache/map_test.go b/cache/map_test.go index 7fb15a8..9ee6683 100644 --- a/cache/map_test.go +++ b/cache/map_test.go @@ -276,3 +276,76 @@ func TestCompareAndSwap_NonExistingKey(t *testing.T) { t.Fatalf("CompareAndSwap on an non-existing key succeeded") } } + +// TestConcurrentClear — adapted from sync/map_test.go. Spawns 10 writers, +// 10 readers, and 10 Clear goroutines; correctness is that nothing panics +// or trips the race detector. +func TestConcurrentClear(t *testing.T) { + var m CacheMap[int] + + var wg sync.WaitGroup + wg.Add(30) + + for i := range 10 { + go func(k, v int) { + defer wg.Done() + m.Store(k, v) + }(i, i*10) + } + for i := range 10 { + go func(k int) { + defer wg.Done() + if _, ok := m.Load(k); ok { + _ = ok + } + }(i) + } + for range 10 { + go func() { + defer wg.Done() + m.Clear() + }() + } + + wg.Wait() + + // After all Clears and Stores have run, the map may be empty or contain a + // subset of the Stores depending on interleaving. Verify no phantom keys + // (keys we never Stored) materialized. + m.Range(func(k int, _ any) bool { + if k < 0 || k >= 10 { + t.Errorf("Range after concurrent Clear/Store saw key %d, never Stored", k) + } + return true + }) +} + +// TestMapClearOneAllocation — adapted from sync/map_test.go. Cache.Clear +// replaces the trie root with one fresh indirect node, which should be the +// only allocation. +func TestMapClearOneAllocation(t *testing.T) { + var m CacheMap[int] + // Prime so the trie has been initialized; Clear of an uninitialized + // trie still calls init which allocates more. + m.Store(0, 0) + allocs := testing.AllocsPerRun(10, func() { + m.Clear() + }) + if allocs > 1 { + t.Errorf("AllocsPerRun of Clear = %v; want 1", allocs) + } +} + +// TestMapRangeNoAllocations — adapted from sync/map_test.go. Range must not +// allocate. +func TestMapRangeNoAllocations(t *testing.T) { + var m CacheMap[int] + allocs := testing.AllocsPerRun(10, func() { + m.Range(func(k int, _ any) bool { + return true + }) + }) + if allocs > 0 { + t.Errorf("AllocsPerRun of Range = %v; want 0", allocs) + } +} diff --git a/cache/trie.go b/cache/trie.go new file mode 100644 index 0000000..0295911 --- /dev/null +++ b/cache/trie.go @@ -0,0 +1,463 @@ +package cache + +import ( + "hash/maphash" + "sync" + "sync/atomic" + "unsafe" +) + +// trie is a concurrent hash-trie map, modeled on Go's internal/sync +// HashTrieMap. It is the backing store for Cache. +// +// Design notes: +// +// - The root is a lazily-allocated indirect node, atomically swapped on +// Clear so readers that already loaded the root pointer keep operating +// on the old tree. +// +// - Interior nodes ("indirect") hold trieBranching atomic child pointers. +// Leaf nodes ("entry") hold a key, an atomic value slot (*V), and an +// atomic overflow pointer for hash-collision chains. +// +// - Load is lock-free: walk the tree with atomic loads, follow overflow +// on hash collision. +// +// - LoadOrStore and Delete take the *parent* indirect node's mutex +// (per-bucket locking) during mutation. After Delete, empty parents +// are pruned bottom-up while locks are held. +// +// - Hash and pointer-size primitives come from userspace: hash/maphash +// for the runtime's typed hasher, unsafe.Sizeof(uintptr(0)) for the +// hash bit budget. No internal/abi, no internal/goarch. + +const ( + trieBranchingLog2 = 4 + trieBranching = 1 << trieBranchingLog2 + trieBranchingMask = trieBranching - 1 +) + +// ptrBits is the number of hash bits consumed per trie walk. Using uintptr +// gives one hasher call on both 32-bit (32 bits consumed) and 64-bit (64 +// bits consumed) platforms; on 32-bit, maphash.Comparable internally calls +// the runtime hasher twice to synthesize 64 bits, of which we keep the low +// uintptr — a cheap extra hash call on rare hardware. +const ptrBits = uint(unsafe.Sizeof(uintptr(0))) * 8 + +// trieNode is the common header for both leaf (entry) and interior +// (indirect) nodes. The layout is carefully matched so a *trieNode can be +// unsafe-cast to either concrete type via the isEntry discriminator. +type trieNode[K comparable, V any] struct { + isEntry bool +} + +// trieIndirect is an interior node with trieBranching atomic child slots. +type trieIndirect[K comparable, V any] struct { + trieNode[K, V] + dead atomic.Bool + mu sync.Mutex + parent *trieIndirect[K, V] + children [trieBranching]atomic.Pointer[trieNode[K, V]] +} + +// trieEntry is a leaf node. The value slot p is an atomic pointer; a nil +// value slot means the entry is tombstoned but still wired into the trie. +// The cache treats a nil slot as terminal — it never stores a value over +// one, and re-creates the key as a fresh entry instead (see entDel in +// cache.go and deleteEntryIf below; resurrecting a tombstone in place +// would invalidate deleteEntryIf's locked predicate). The overflow chain +// handles hash-equal keys (full hash collisions). +type trieEntry[K comparable, V any] struct { + trieNode[K, V] + overflow atomic.Pointer[trieEntry[K, V]] + key K + p atomic.Pointer[V] +} + +func newTrieIndirect[K comparable, V any](parent *trieIndirect[K, V]) *trieIndirect[K, V] { + n := &trieIndirect[K, V]{parent: parent} + n.isEntry = false + return n +} + +func newTrieEntry[K comparable, V any](key K, value *V) *trieEntry[K, V] { + e := &trieEntry[K, V]{key: key} + e.isEntry = true + if value != nil { + e.p.Store(value) + } + return e +} + +// entry unsafe-casts a trieNode to its leaf form. Every call site checks +// n.isEntry first, so no runtime guard is needed here. +func (n *trieNode[K, V]) entry() *trieEntry[K, V] { + return (*trieEntry[K, V])(unsafe.Pointer(n)) +} + +// indirect unsafe-casts a trieNode to its interior form. Every call site +// checks !n.isEntry first, so no runtime guard is needed here. +func (n *trieNode[K, V]) indirect() *trieIndirect[K, V] { + return (*trieIndirect[K, V])(unsafe.Pointer(n)) +} + +// empty reports whether all child slots of i are nil. +func (i *trieIndirect[K, V]) empty() bool { + for j := range i.children { + if i.children[j].Load() != nil { + return false + } + } + return true +} + +type trie[K comparable, V any] struct { + inited atomic.Bool + initMu sync.Mutex + root atomic.Pointer[trieIndirect[K, V]] + seed maphash.Seed + + // hashFn, if non-nil, overrides maphash.Comparable for tests that need + // to construct hash collisions deterministically. Never set in + // production code. + hashFn func(K) uintptr +} + +func (t *trie[K, V]) init() { + if t.inited.Load() { + return + } + t.initMu.Lock() + defer t.initMu.Unlock() + if t.inited.Load() { + return + } + if t.hashFn == nil { + t.seed = maphash.MakeSeed() + } + t.root.Store(newTrieIndirect[K, V](nil)) + t.inited.Store(true) +} + +func (t *trie[K, V]) hash(k K) uintptr { + if t.hashFn != nil { + return t.hashFn(k) + } + return uintptr(maphash.Comparable(t.seed, k)) +} + +// loadEntry returns the entry for k, or nil if the key is not present. The +// lookup is lock-free: atomic loads walk the tree, then a final walk over +// the overflow chain checks for an exact key match. +func (t *trie[K, V]) loadEntry(k K) *trieEntry[K, V] { + if !t.inited.Load() { + return nil + } + h := t.hash(k) + i := t.root.Load() + shift := ptrBits + for shift != 0 { + shift -= trieBranchingLog2 + n := i.children[(h>>shift)&trieBranchingMask].Load() + if n == nil { + return nil + } + if n.isEntry { + for e := n.entry(); e != nil; e = e.overflow.Load() { + if e.key == k { + return e + } + } + return nil + } + i = n.indirect() + } + // Unreachable: the tree depth is bounded by ptrBits / trieBranchingLog2 + // (16 levels on 64-bit, 8 on 32-bit), so the loop always returns before + // shift reaches 0. Kept as a defensive assertion in case the tree is + // ever corrupted. + panic("cache: trie ran out of hash bits in loadEntry") +} + +// loadOrStoreEntry returns the existing entry for k if present. Otherwise it +// inserts a new entry initialized with value (which may be nil, meaning the +// caller will fill the value slot later) and returns it. The second return +// is true if an existing entry was loaded, false if a new one was stored. +// +// The insertion path takes the parent indirect node's lock. On hash prefix +// collision (two distinct keys sharing the same hash bits at the current +// depth) we build out indirection nodes until the bits diverge, via expand. +func (t *trie[K, V]) loadOrStoreEntry(k K, value *V) (*trieEntry[K, V], bool) { + t.init() + h := t.hash(k) + var i *trieIndirect[K, V] + var shift uint + var slot *atomic.Pointer[trieNode[K, V]] + var n *trieNode[K, V] + for { + i = t.root.Load() + shift = ptrBits + haveInsertPoint := false + for shift != 0 { + shift -= trieBranchingLog2 + slot = &i.children[(h>>shift)&trieBranchingMask] + n = slot.Load() + if n == nil { + haveInsertPoint = true + break + } + if n.isEntry { + for e := n.entry(); e != nil; e = e.overflow.Load() { + if e.key == k { + return e, true + } + } + haveInsertPoint = true + break + } + i = n.indirect() + } + if !haveInsertPoint { + // Unreachable: see the note in loadEntry above. + panic("cache: trie ran out of hash bits in loadOrStoreEntry") + } + i.mu.Lock() + n = slot.Load() + if (n == nil || n.isEntry) && !i.dead.Load() { + break + } + i.mu.Unlock() + } + defer i.mu.Unlock() + + var oldEntry *trieEntry[K, V] + if n != nil { + oldEntry = n.entry() + for e := oldEntry; e != nil; e = e.overflow.Load() { + if e.key == k { + return e, true + } + } + } + + newEntry := newTrieEntry(k, value) + if oldEntry == nil { + slot.Store(&newEntry.trieNode) + } else { + slot.Store(t.expand(oldEntry, newEntry, h, shift, i)) + } + return newEntry, false +} + +// expand resolves an insert where a leaf already sits at the target slot +// but holds a different key. If the old and new keys hash identically, we +// chain the old entry onto the new entry's overflow list. Otherwise we +// build out indirect nodes until the next differing hash bit, then place +// both entries in distinct children. +func (t *trie[K, V]) expand(oldEntry, newEntry *trieEntry[K, V], newHash uintptr, shift uint, parent *trieIndirect[K, V]) *trieNode[K, V] { + oldHash := t.hash(oldEntry.key) + if oldHash == newHash { + newEntry.overflow.Store(oldEntry) + return &newEntry.trieNode + } + top := newTrieIndirect(parent) + cur := top + for { + if shift == 0 { + // Unreachable: oldHash and newHash differ (same-hash case + // returned above via the overflow chain), so some bit in + // [0, shift) must differ and the loop exits before shift=0. + panic("cache: trie ran out of hash bits in expand") + } + shift -= trieBranchingLog2 + oi := (oldHash >> shift) & trieBranchingMask + ni := (newHash >> shift) & trieBranchingMask + if oi != ni { + cur.children[oi].Store(&oldEntry.trieNode) + cur.children[ni].Store(&newEntry.trieNode) + break + } + next := newTrieIndirect(cur) + cur.children[oi].Store(&next.trieNode) + cur = next + } + return &top.trieNode +} + +// deleteEntry removes the entry for k from the trie. Returns the removed +// entry (or nil). Empty interior ancestors are pruned bottom-up. +func (t *trie[K, V]) deleteEntry(k K) *trieEntry[K, V] { + return t.deleteEntryIf(k, nil) +} + +// deleteEntryIf is like deleteEntry, but only proceeds if pred(entry) +// returns true when evaluated under the parent's lock. Cache uses this to +// physically unlink only tombstoned entries (nil value slot). The value +// slot of a tombstoned entry never transitions back to non-nil (see entDel +// in cache.go), so a nil observed by pred under the lock cannot be +// invalidated by a concurrent writer; and if the key was deleted and +// re-created in the meantime, the lookup finds the fresh live entry and +// pred leaves it alone. +// +// If pred is nil, deleteEntryIf is equivalent to unconditional deletion. +func (t *trie[K, V]) deleteEntryIf(k K, pred func(*trieEntry[K, V]) bool) *trieEntry[K, V] { + if !t.inited.Load() { + return nil + } + h := t.hash(k) + + // Walk to the entry, then lock the parent indirect node. We retry if we + // see the parent has been marked dead by concurrent pruning. + var i *trieIndirect[K, V] + var shift uint + var slot *atomic.Pointer[trieNode[K, V]] + var n *trieNode[K, V] + for { + i = t.root.Load() + shift = ptrBits + found := false + for shift != 0 { + shift -= trieBranchingLog2 + slot = &i.children[(h>>shift)&trieBranchingMask] + n = slot.Load() + if n == nil { + return nil + } + if n.isEntry { + found = true + break + } + i = n.indirect() + } + if !found { + // Unreachable: see the note in loadEntry above. + panic("cache: trie ran out of hash bits in deleteEntry") + } + i.mu.Lock() + if i.dead.Load() { + i.mu.Unlock() + continue + } + n = slot.Load() + if n == nil { + // The entry was deleted concurrently; nothing to do. + i.mu.Unlock() + return nil + } + if !n.isEntry { + // The slot expanded into an indirect node between our + // unlocked walk and the lock (a concurrent insert with a + // colliding hash prefix built out this level). The key may + // now live deeper in the new subtree; retry from the root, + // as internal/sync.HashTrieMap's find does. + i.mu.Unlock() + continue + } + break + } + + // Under i.mu, locate the entry matching k in the overflow chain, call + // pred, and unlink in one pass. Because we pre-locate the target before + // calling pred, the subsequent removal cannot fail to find k. + head := n.entry() + var ( + target *trieEntry[K, V] + prev *trieEntry[K, V] // non-nil when target is not the head + newHead *trieEntry[K, V] // chain after removal, nil if slot becomes empty + ) + if head.key == k { + target = head + } else { + prev = head + for { + next := prev.overflow.Load() + if next == nil { + // k is not in the chain. + i.mu.Unlock() + return nil + } + if next.key == k { + target = next + break + } + prev = next + } + } + if pred != nil && !pred(target) { + i.mu.Unlock() + return nil + } + if prev == nil { + // Removing head. + newHead = target.overflow.Load() + } else { + prev.overflow.Store(target.overflow.Load()) + newHead = head + } + if newHead != nil { + slot.Store(&newHead.trieNode) + i.mu.Unlock() + return target + } + slot.Store(nil) + + // Prune empty parents. Walk up while holding locks as a hand-over-hand + // from child to parent. + for i.parent != nil && i.empty() { + if shift == ptrBits { + // Unreachable: pruning walks strictly upward from a leaf, so + // shift is always < ptrBits when we enter (we consumed at + // least trieBranchingLog2 bits on descent). Kept as a + // defensive assertion. + panic("cache: trie shift overflow in deleteEntry pruning") + } + shift += trieBranchingLog2 + parent := i.parent + parent.mu.Lock() + i.dead.Store(true) + parent.children[(h>>shift)&trieBranchingMask].Store(nil) + i.mu.Unlock() + i = parent + } + i.mu.Unlock() + return target +} + +// walk iterates every entry in the trie, calling f for each. If f returns +// false, iteration stops. No snapshot is taken; concurrent mutations may +// or may not be observed. A given key is never visited more than once. +func (t *trie[K, V]) walk(f func(*trieEntry[K, V]) bool) { + if !t.inited.Load() { + return + } + trieWalkIndirect(t.root.Load(), f) +} + +func trieWalkIndirect[K comparable, V any](i *trieIndirect[K, V], f func(*trieEntry[K, V]) bool) bool { + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + if !n.isEntry { + if !trieWalkIndirect(n.indirect(), f) { + return false + } + continue + } + for e := n.entry(); e != nil; e = e.overflow.Load() { + if !f(e) { + return false + } + } + } + return true +} + +// clear replaces the root with a fresh empty indirect node. Concurrent +// readers that already loaded the old root keep walking it; the old tree +// becomes unreachable once all such readers finish. +func (t *trie[K, V]) clear() { + t.init() + t.root.Store(newTrieIndirect[K, V](nil)) +} diff --git a/cache/trie_test.go b/cache/trie_test.go new file mode 100644 index 0000000..b0e374c --- /dev/null +++ b/cache/trie_test.go @@ -0,0 +1,681 @@ +package cache + +import ( + "fmt" + "math/rand/v2" + "sync" + "sync/atomic" + "testing" +) + +// Helper: store or replace the value slot with a fresh pointer carrying v. +// The trie owns the entry; the caller owns the value pointer identity. +func triePut[K comparable, V any](t *trie[K, V], k K, v V) *V { + p := &v + e, _ := t.loadOrStoreEntry(k, p) + e.p.Store(p) + return p +} + +func trieGet[K comparable, V any](t *trie[K, V], k K) (V, bool) { + e := t.loadEntry(k) + if e == nil { + return *new(V), false + } + p := e.p.Load() + if p == nil { + return *new(V), false + } + return *p, true +} + +// verifyTrie checks structural invariants once concurrent operations have +// quiesced: every entry hangs at the position prescribed by its hash, +// overflow chains hold only full-hash collisions, parent pointers are +// consistent, no linked node is marked dead, and no non-root indirect is +// empty (pruning must have removed it). It must not run concurrently with +// mutations. +func verifyTrie[K comparable, V any](t *testing.T, tr *trie[K, V]) { + t.Helper() + if !tr.inited.Load() { + return + } + root := tr.root.Load() + var walk func(i *trieIndirect[K, V], prefix uintptr, nbits uint) + walk = func(i *trieIndirect[K, V], prefix uintptr, nbits uint) { + if i.dead.Load() { + t.Fatalf("linked indirect at depth %d is marked dead", nbits/trieBranchingLog2) + } + if i != root && i.empty() { + t.Fatalf("empty non-root indirect linked at depth %d (pruning missed it)", nbits/trieBranchingLog2) + } + for j := range i.children { + n := i.children[j].Load() + if n == nil { + continue + } + cp := prefix< ptrBits { + t.Fatal("trie deeper than the hash bit budget") + } + if n.isEntry { + head := n.entry() + headHash := tr.hash(head.key) + for e := head; e != nil; e = e.overflow.Load() { + h := tr.hash(e.key) + if h>>(ptrBits-cb) != cp { + t.Fatalf("entry for key %v misplaced: hash %#x does not match its path %#x at depth %d", + e.key, h, cp, cb/trieBranchingLog2) + } + if h != headHash { + t.Fatalf("overflow chain mixes hashes: key %v hash %#x vs head %v hash %#x", + e.key, h, head.key, headHash) + } + } + } else { + ci := n.indirect() + if ci.parent != i { + t.Fatalf("indirect at depth %d has a wrong parent pointer", cb/trieBranchingLog2) + } + walk(ci, cp, cb) + } + } + } + walk(root, 0, 0) +} + +func TestTrie_ZeroValueLoad(t *testing.T) { + var tr trie[string, int] + if e := tr.loadEntry("missing"); e != nil { + t.Fatalf("loadEntry on zero-value trie returned %v, want nil", e) + } + var visited int + tr.walk(func(*trieEntry[string, int]) bool { visited++; return true }) + if visited != 0 { + t.Fatalf("walk on zero-value trie visited %d, want 0", visited) + } + if e := tr.deleteEntry("missing"); e != nil { + t.Fatalf("deleteEntry on zero-value trie returned %v, want nil", e) + } +} + +func TestTrie_StoreLoad(t *testing.T) { + var tr trie[string, int] + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + if v, ok := trieGet(&tr, "a"); !ok || v != 1 { + t.Fatalf("get a: v=%d ok=%v, want 1 true", v, ok) + } + if v, ok := trieGet(&tr, "b"); !ok || v != 2 { + t.Fatalf("get b: v=%d ok=%v, want 2 true", v, ok) + } + if _, ok := trieGet(&tr, "c"); ok { + t.Fatal("get c should be missing") + } +} + +func TestTrie_LoadOrStoreExisting(t *testing.T) { + var tr trie[string, int] + v1 := 1 + e1, loaded := tr.loadOrStoreEntry("k", &v1) + if loaded { + t.Fatal("first loadOrStoreEntry should not report loaded") + } + v2 := 2 + e2, loaded := tr.loadOrStoreEntry("k", &v2) + if !loaded { + t.Fatal("second loadOrStoreEntry should report loaded") + } + if e1 != e2 { + t.Fatal("loadOrStoreEntry should return the same entry object on subsequent calls") + } + // The stored value is v1's pointer; v2 was ignored. + if got := *e2.p.Load(); got != 1 { + t.Fatalf("stored value = %d, want 1 (new value must not overwrite)", got) + } +} + +func TestTrie_LoadOrStoreWithNilValue(t *testing.T) { + // Cache always creates entries with a value in place (a nil slot means + // tombstoned), but the trie itself is value-agnostic: verify + // loadOrStoreEntry(k, nil) is legal and the entry.p is nil. + var tr trie[string, int] + e, loaded := tr.loadOrStoreEntry("k", nil) + if loaded { + t.Fatal("expected not-loaded for first insert") + } + if e.p.Load() != nil { + t.Fatal("nil-initialized entry should have nil value slot") + } + v := 42 + e.p.Store(&v) + if got, ok := trieGet(&tr, "k"); !ok || got != 42 { + t.Fatalf("after Store: got=%d ok=%v, want 42 true", got, ok) + } +} + +func TestTrie_Delete(t *testing.T) { + var tr trie[string, int] + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + + e := tr.deleteEntry("a") + if e == nil || e.key != "a" { + t.Fatalf("deleteEntry(a) = %v, want entry for a", e) + } + if _, ok := trieGet(&tr, "a"); ok { + t.Fatal("after delete, a should be missing") + } + if _, ok := trieGet(&tr, "b"); !ok { + t.Fatal("after delete of a, b should still be present") + } + if e := tr.deleteEntry("a"); e != nil { + t.Fatalf("second deleteEntry(a) = %v, want nil", e) + } +} + +func TestTrie_DeletePrunesEmptyParents(t *testing.T) { + // Insert many keys so the tree grows, then delete them all. After, the + // root should be empty (all children nil). + var tr trie[int, int] + const n = 1000 + for i := range n { + triePut(&tr, i, i) + } + for i := range n { + if e := tr.deleteEntry(i); e == nil { + t.Fatalf("deleteEntry(%d) unexpectedly returned nil", i) + } + } + root := tr.root.Load() + for i := range root.children { + if n := root.children[i].Load(); n != nil { + t.Fatalf("root.children[%d] = %v, want nil (pruning should have cleared)", i, n) + } + } +} + +func TestTrie_Walk(t *testing.T) { + var tr trie[int, int] + const n = 256 + for i := range n { + triePut(&tr, i, i*10) + } + seen := make(map[int]int) + tr.walk(func(e *trieEntry[int, int]) bool { + if p := e.p.Load(); p != nil { + seen[e.key] = *p + } + return true + }) + if len(seen) != n { + t.Fatalf("walk saw %d entries, want %d", len(seen), n) + } + for k, v := range seen { + if v != k*10 { + t.Fatalf("seen[%d] = %d, want %d", k, v, k*10) + } + } +} + +func TestTrie_WalkEarlyStop(t *testing.T) { + var tr trie[int, int] + for i := range 100 { + triePut(&tr, i, i) + } + var count int + tr.walk(func(*trieEntry[int, int]) bool { + count++ + return count < 5 + }) + if count != 5 { + t.Fatalf("walk visited %d, want 5 (early stop)", count) + } +} + +func TestTrie_Clear(t *testing.T) { + var tr trie[int, int] + for i := range 100 { + triePut(&tr, i, i) + } + tr.clear() + for i := range 100 { + if _, ok := trieGet(&tr, i); ok { + t.Fatalf("after clear, key %d should be missing", i) + } + } + var count int + tr.walk(func(*trieEntry[int, int]) bool { count++; return true }) + if count != 0 { + t.Fatalf("walk after clear visited %d, want 0", count) + } + // Trie is still usable after clear. + triePut(&tr, 1, 1) + if v, ok := trieGet(&tr, 1); !ok || v != 1 { + t.Fatalf("reuse after clear: v=%d ok=%v", v, ok) + } +} + +// TestTrie_ManyKeys exercises the expand path repeatedly: random keys will +// routinely collide on short hash prefixes, forcing the trie to extend its +// depth. 20k keys is enough that most hash prefixes of length 4 will +// collide multiple times. +func TestTrie_ManyKeys(t *testing.T) { + var tr trie[int, int] + const n = 20_000 + keys := rand.Perm(n) + for _, k := range keys { + triePut(&tr, k, k*2) + } + for _, k := range keys { + v, ok := trieGet(&tr, k) + if !ok || v != k*2 { + t.Fatalf("lookup(%d): v=%d ok=%v, want %d true", k, v, ok, k*2) + } + } + // Delete half and re-verify. + for i, k := range keys { + if i%2 == 0 { + tr.deleteEntry(k) + } + } + for i, k := range keys { + v, ok := trieGet(&tr, k) + switch i % 2 { + case 0: + if ok { + t.Fatalf("deleted key %d still present: v=%d", k, v) + } + case 1: + if !ok || v != k*2 { + t.Fatalf("surviving key %d: v=%d ok=%v, want %d true", k, v, ok, k*2) + } + } + } + verifyTrie(t, &tr) +} + +// TestTrie_FullHashCollision verifies the overflow-chain behavior by forcing +// two distinct keys to hash identically via the test-only hashFn hook. +func TestTrie_FullHashCollision(t *testing.T) { + var tr trie[string, int] + // Collide everything in a single hash bucket. + tr.hashFn = func(string) uintptr { return 0xdeadbeef } + + const n = 10 + for i := range n { + k := fmt.Sprintf("k%d", i) + triePut(&tr, k, i) + } + // All n entries share one overflow chain under the same leaf slot. + for i := range n { + k := fmt.Sprintf("k%d", i) + v, ok := trieGet(&tr, k) + if !ok || v != i { + t.Fatalf("get %q: v=%d ok=%v, want %d true", k, v, ok, i) + } + } + // Delete one from the middle; the rest remain. + tr.deleteEntry("k5") + if _, ok := trieGet(&tr, "k5"); ok { + t.Fatal("k5 should be deleted") + } + for i := range n { + if i == 5 { + continue + } + k := fmt.Sprintf("k%d", i) + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%q should survive delete of k5", k) + } + } +} + +// TestTrie_FullHashCollisionDeleteHead deletes the head of the overflow +// chain. The chain head is what the slot points at, so slot must be +// updated to the next entry. +func TestTrie_FullHashCollisionDeleteHead(t *testing.T) { + var tr trie[string, int] + tr.hashFn = func(string) uintptr { return 0x42 } + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + triePut(&tr, "c", 3) + + // Delete the most-recently-inserted (head of chain, per expand logic). + if e := tr.deleteEntry("c"); e == nil || e.key != "c" { + t.Fatalf("delete c: %v", e) + } + for _, k := range []string{"a", "b"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%s missing after delete of chain head", k) + } + } +} + +func TestTrie_LoadOrStoreDuringCollision(t *testing.T) { + // After a collision chain exists, loadOrStoreEntry for an already-present + // key must find it via the overflow walk, not append a duplicate. + var tr trie[string, int] + tr.hashFn = func(string) uintptr { return 0xabcd } + triePut(&tr, "x", 1) + triePut(&tr, "y", 2) + v := 999 + e, loaded := tr.loadOrStoreEntry("x", &v) + if !loaded { + t.Fatal("second loadOrStoreEntry(x) should report loaded") + } + if got := *e.p.Load(); got != 1 { + t.Fatalf("existing entry value=%d, want 1 (new value must be ignored)", got) + } +} + +// TestTrie_ConcurrentStoreLoad runs many goroutines performing +// loadOrStoreEntry and loadEntry over disjoint key ranges, verifying no +// race and eventual consistency. +func TestTrie_ConcurrentStoreLoad(t *testing.T) { + var tr trie[int, int] + const workers = 8 + const perWorker = 2000 + + var wg sync.WaitGroup + wg.Add(workers) + for w := range workers { + go func(w int) { + defer wg.Done() + base := w * perWorker + for i := range perWorker { + triePut(&tr, base+i, base+i) + } + for i := range perWorker { + v, ok := trieGet(&tr, base+i) + if !ok || v != base+i { + t.Errorf("worker %d, key %d: v=%d ok=%v", w, base+i, v, ok) + return + } + } + }(w) + } + wg.Wait() + verifyTrie(t, &tr) +} + +// TestTrie_ConcurrentSameKey hammers the same key with many goroutines +// doing loadOrStoreEntry. Exactly one must report loaded=false; all others +// must observe the same entry object. +func TestTrie_ConcurrentSameKey(t *testing.T) { + const workers = 64 + for iter := range 100 { + var tr trie[int, int] + var wg sync.WaitGroup + wg.Add(workers) + entries := make([]*trieEntry[int, int], workers) + loadeds := make([]bool, workers) + for w := range workers { + go func(w int) { + defer wg.Done() + v := w + e, loaded := tr.loadOrStoreEntry(iter, &v) + entries[w] = e + loadeds[w] = loaded + }(w) + } + wg.Wait() + + var inserted int + for _, l := range loadeds { + if !l { + inserted++ + } + } + if inserted != 1 { + t.Fatalf("iter %d: %d inserters, want exactly 1", iter, inserted) + } + first := entries[0] + for i, e := range entries { + if e != first { + t.Fatalf("iter %d: worker %d saw entry %p, want %p", iter, i, e, first) + } + } + verifyTrie(t, &tr) + } +} + +// TestTrie_ConcurrentDeletePrune exercises concurrent deletes across many +// keys so parent pruning fires from many angles simultaneously. +func TestTrie_ConcurrentDeletePrune(t *testing.T) { + var tr trie[int, int] + const n = 10_000 + for i := range n { + triePut(&tr, i, i) + } + const workers = 8 + var wg sync.WaitGroup + wg.Add(workers) + for w := range workers { + go func(w int) { + defer wg.Done() + for i := w; i < n; i += workers { + tr.deleteEntry(i) + } + }(w) + } + wg.Wait() + for i := range n { + if _, ok := trieGet(&tr, i); ok { + t.Fatalf("key %d should be deleted", i) + } + } + // After all deletions, root should be empty. + if root := tr.root.Load(); !root.empty() { + t.Fatal("root not pruned: has surviving children") + } + verifyTrie(t, &tr) +} + +// TestTrie_RangeDuringMutation runs walk concurrently with store and +// delete. No snapshot is promised; we assert only no-crash + that every +// observed key-value pair is plausible (key was stored, value is key*2 or +// nil). +func TestTrie_RangeDuringMutation(t *testing.T) { + var tr trie[int, int] + const n = 2000 + for i := range n { + triePut(&tr, i, i*2) + } + + var stop atomic.Bool + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + i := rand.IntN(n) + triePut(&tr, i, i*2) + tr.deleteEntry(i) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + tr.walk(func(e *trieEntry[int, int]) bool { + p := e.p.Load() + if p != nil && *p != e.key*2 { + t.Errorf("invariant break: key=%d value=%d", e.key, *p) + return false + } + return true + }) + } + }() + + // Let the race run briefly. + for range 50_000 { + i := rand.IntN(n) + if _, ok := trieGet(&tr, i); ok { + // just a read + } + } + stop.Store(true) + wg.Wait() + verifyTrie(t, &tr) +} + +// TestTrie_DeleteMissingInCollisionChain walks an overflow chain in search +// of a key that is never present, exercising deleteEntryIf's +// "walk the chain to the end without finding k" branch. +func TestTrie_DeleteMissingInCollisionChain(t *testing.T) { + var tr trie[string, int] + tr.hashFn = func(string) uintptr { return 0x1234 } + triePut(&tr, "a", 1) + triePut(&tr, "b", 2) + triePut(&tr, "c", 3) + // "missing" collides with a/b/c but was never stored. + if e := tr.deleteEntry("missing"); e != nil { + t.Fatalf("deleteEntry of never-stored colliding key returned %v, want nil", e) + } + // Original entries survive. + for _, k := range []string{"a", "b", "c"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("%s should survive delete of missing colliding key", k) + } + } +} + +// TestTrie_DeleteRaceRetries stresses deleteEntry's lock-then-retry paths: +// the "parent is dead" retry and the "slot became nil after lock" +// early-return. Both fire when a concurrent delete prunes or empties the +// slot between our unlocked walk and our mu.Lock. +// +// The stress needs sustained concurrent deletes on many hash-adjacent +// keys so that pruning and slot clearing happen continuously. +func TestTrie_DeleteRaceRetries(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + const workers = 16 + const keySpace = 256 + for range 200 { + var tr trie[int, int] + // Prime: every key is present. + for i := range keySpace { + triePut(&tr, i, i) + } + var wg sync.WaitGroup + wg.Add(workers * 2) + // Deleter swarms. + for w := range workers { + go func(w int) { + defer wg.Done() + for i := w; i < keySpace; i += workers { + tr.deleteEntry(i) + } + }(w) + } + // Concurrent deleter attempting same keys (likely to race on lock). + for w := range workers { + go func(w int) { + defer wg.Done() + for i := keySpace - 1 - w; i >= 0; i -= workers { + tr.deleteEntry(i) + } + }(w) + } + wg.Wait() + verifyTrie(t, &tr) + } +} + +// TestTrie_DeleteRaceAgainstExpand targets deleteEntryIf's "slot changed to +// non-entry after lock" branch. An indirect-node split (expand) during the +// window between delete's unlocked walk and its parent.Lock forces the +// reload to see a non-entry node, which must trigger a retry from the root +// (matching internal/sync.HashTrieMap's find), not a give-up: "a" is present +// for the whole round until the delete removes it, so deleteEntry("a") must +// always find and return it. +// +// We use a controlled hashFn that collides a few keys in the top bits but +// not all bits, so inserting them triggers expand. Concurrent insert+delete +// races through the expand window. +func TestTrie_DeleteRaceAgainstExpand(t *testing.T) { + if testing.Short() { + t.Skip("skipping stress test in -short") + } + // Keys whose hashes match in the top 4 bits but diverge later: they + // share a slot at depth 1 and a leaf at depth 2 (via overflow then + // expand). The constants are built by shifting so they fit a uintptr + // on both 64-bit and 32-bit platforms. + hashes := map[string]uintptr{ + "a": 0xA << (ptrBits - 4), + "b": 0xA<<(ptrBits-4) | 1, + "c": 0xA<<(ptrBits-4) | 2, + "d": 0xB << (ptrBits - 4), + } + for i := range 20_000 { + var tr trie[string, int] + tr.hashFn = func(k string) uintptr { return hashes[k] } + triePut(&tr, "a", 1) + triePut(&tr, "d", 4) + + var wg sync.WaitGroup + wg.Add(3) + var deleted *trieEntry[string, int] + go func() { + defer wg.Done() + triePut(&tr, "b", 2) + }() + go func() { + defer wg.Done() + triePut(&tr, "c", 3) + }() + go func() { + defer wg.Done() + deleted = tr.deleteEntry("a") + }() + wg.Wait() + + if deleted == nil || deleted.key != "a" { + t.Fatalf("round %d: deleteEntry(a) = %v, want the entry for a (delete must retry across a concurrent expand)", i, deleted) + } + if e := tr.loadEntry("a"); e != nil { + t.Fatalf("round %d: a still present after deleteEntry returned it", i) + } + for _, k := range []string{"b", "c", "d"} { + if _, ok := trieGet(&tr, k); !ok { + t.Fatalf("round %d: %s missing after delete of a", i, k) + } + } + verifyTrie(t, &tr) + } +} + +// TestTrie_DeleteDuringConcurrentStore: a goroutine repeatedly deletes key +// K while another repeatedly stores it. Both succeed; the final state is +// "either present or absent" and neither crashes. +func TestTrie_DeleteDuringConcurrentStore(t *testing.T) { + var tr trie[int, int] + var stop atomic.Bool + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for !stop.Load() { + triePut(&tr, 7, 42) + } + }() + go func() { + defer wg.Done() + for !stop.Load() { + tr.deleteEntry(7) + } + }() + for range 100_000 { + trieGet(&tr, 7) + } + stop.Store(true) + wg.Wait() + verifyTrie(t, &tr) +} diff --git a/cache/wrappers_test.go b/cache/wrappers_test.go new file mode 100644 index 0000000..9e5eaa1 --- /dev/null +++ b/cache/wrappers_test.go @@ -0,0 +1,305 @@ +package cache + +import ( + "errors" + "sort" + "sync/atomic" + "testing" + "time" +) + +// Item delegates to Cache[struct{}, V]. These tests exercise every method so +// the thin wrappers contribute to coverage and so a refactor of the underlying +// Cache that forgets to propagate a method would be caught. + +func TestItem_BasicGetAndCaches(t *testing.T) { + i := NewItem[int]() + var calls int32 + v, err, s := i.Get(func() (int, error) { + atomic.AddInt32(&calls, 1) + return 42, nil + }) + if v != 42 || err != nil || !s.IsMiss() { + t.Fatalf("first Get: got v=%d err=%v s=%v, want v=42 err=nil Miss", v, err, s) + } + v, err, s = i.Get(func() (int, error) { + atomic.AddInt32(&calls, 1) + return -1, nil + }) + if v != 42 || err != nil || !s.IsHit() { + t.Fatalf("second Get: got v=%d err=%v s=%v, want v=42 err=nil Hit", v, err, s) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("miss fn called %d times, want 1", got) + } +} + +func TestItem_TryGet(t *testing.T) { + i := NewItem[string]() + _, _, s := i.TryGet() + if !s.IsMiss() { + t.Fatalf("TryGet on empty item: state=%v want Miss", s) + } + i.Set("hello") + v, err, s := i.TryGet() + if v != "hello" || err != nil || !s.IsHit() { + t.Fatalf("TryGet after Set: v=%q err=%v s=%v", v, err, s) + } +} + +func TestItem_Delete(t *testing.T) { + i := NewItem[int]() + i.Set(7) + v, err, s := i.Delete() + if v != 7 || err != nil || !s.IsHit() { + t.Fatalf("Delete of set value: v=%d err=%v s=%v", v, err, s) + } + _, _, s = i.TryGet() + if !s.IsMiss() { + t.Fatalf("TryGet after Delete: state=%v want Miss", s) + } + // Deleting an already-empty item is a no-op. + _, _, s = i.Delete() + if !s.IsMiss() { + t.Fatalf("Delete of empty item: state=%v want Miss", s) + } +} + +func TestItem_Expire(t *testing.T) { + i := NewItem[int](MaxAge(time.Hour)) + i.Set(99) + if _, _, s := i.TryGet(); !s.IsHit() { + t.Fatalf("before Expire: want Hit") + } + i.Expire() + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after Expire: want Miss") + } +} + +func TestItem_Swap(t *testing.T) { + i := NewItem[int]() + old, err, s := i.Swap(5) + if err != nil || !s.IsMiss() { + t.Fatalf("first Swap: old=%d err=%v s=%v, want Miss", old, err, s) + } + old, err, s = i.Swap(6) + if old != 5 || err != nil || !s.IsHit() { + t.Fatalf("second Swap: old=%d err=%v s=%v, want 5 Hit", old, err, s) + } + if v, _, _ := i.TryGet(); v != 6 { + t.Fatalf("after second Swap: TryGet=%d, want 6", v) + } +} + +func TestItem_CompareAndSwap(t *testing.T) { + i := NewItem[int]() + i.Set(10) + if !i.CompareAndSwap(10, 11) { + t.Fatal("CAS 10->11 should succeed") + } + if i.CompareAndSwap(10, 12) { + t.Fatal("CAS 10->12 should fail after value is 11") + } + if v, _, _ := i.TryGet(); v != 11 { + t.Fatalf("after CAS: TryGet=%d, want 11", v) + } +} + +func TestItem_CompareAndDelete(t *testing.T) { + i := NewItem[int]() + i.Set(7) + if i.CompareAndDelete(8) { + t.Fatal("CAD of wrong value should fail") + } + if !i.CompareAndDelete(7) { + t.Fatal("CAD of correct value should succeed") + } + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after CAD: state=%v want Miss", s) + } +} + +func TestItem_Clear(t *testing.T) { + i := NewItem[int]() + i.Set(1) + i.Clear() + if _, _, s := i.TryGet(); !s.IsMiss() { + t.Fatalf("after Clear: state=%v want Miss", s) + } +} + +func TestItem_ZeroValue(t *testing.T) { + // Docstring says zero-value Item is valid. + var i Item[int] + v, err, s := i.Get(func() (int, error) { return 3, nil }) + if v != 3 || err != nil || !s.IsMiss() { + t.Fatalf("zero Item Get: v=%d err=%v s=%v", v, err, s) + } + if v, _, _ := i.TryGet(); v != 3 { + t.Fatalf("zero Item TryGet: v=%d want 3", v) + } +} + +// Set is a key-only cache over Cache[K, struct{}]. + +func TestSet_BasicGet(t *testing.T) { + s := NewSet[string]() + var calls int32 + err, ks := s.Get("a", func() error { + atomic.AddInt32(&calls, 1) + return nil + }) + if err != nil || !ks.IsMiss() { + t.Fatalf("first Get: err=%v state=%v want nil Miss", err, ks) + } + err, ks = s.Get("a", func() error { + atomic.AddInt32(&calls, 1) + return nil + }) + if err != nil || !ks.IsHit() { + t.Fatalf("second Get: err=%v state=%v want nil Hit", err, ks) + } + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("miss fn called %d times, want 1", got) + } +} + +func TestSet_GetError(t *testing.T) { + s := NewSet[string]() + wantErr := errors.New("boom") + err, ks := s.Get("a", func() error { return wantErr }) + if !errors.Is(err, wantErr) || !ks.IsMiss() { + t.Fatalf("Get with error: err=%v state=%v", err, ks) + } +} + +func TestSet_TryGet(t *testing.T) { + s := NewSet[string]() + _, ks := s.TryGet("missing") + if !ks.IsMiss() { + t.Fatalf("TryGet missing: state=%v", ks) + } + s.Set("k") + err, ks := s.TryGet("k") + if err != nil || !ks.IsHit() { + t.Fatalf("TryGet after Set: err=%v state=%v", err, ks) + } +} + +func TestSet_Delete(t *testing.T) { + s := NewSet[string]() + s.Set("k") + _, ks := s.Delete("k") + if !ks.IsHit() { + t.Fatalf("Delete existing: state=%v want Hit", ks) + } + _, ks = s.Delete("k") + if !ks.IsMiss() { + t.Fatalf("Delete missing: state=%v want Miss", ks) + } +} + +func TestSet_Expire(t *testing.T) { + s := NewSet[string](MaxAge(time.Hour)) + s.Set("k") + s.Expire("k") + if _, ks := s.TryGet("k"); !ks.IsMiss() { + t.Fatalf("TryGet after Expire: state=%v want Miss", ks) + } +} + +func TestSet_Range(t *testing.T) { + s := NewSet[string]() + for _, k := range []string{"a", "b", "c"} { + s.Set(k) + } + var got []string + s.Range(func(k string, err error) bool { + if err != nil { + t.Errorf("unexpected err for key %q: %v", k, err) + } + got = append(got, k) + return true + }) + sort.Strings(got) + want := []string{"a", "b", "c"} + if len(got) != len(want) { + t.Fatalf("Range yielded %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("Range yielded %v, want %v", got, want) + } + } + + // Early stop. + count := 0 + s.Range(func(string, error) bool { + count++ + return false + }) + if count != 1 { + t.Fatalf("Range early-stop visited %d, want 1", count) + } +} + +func TestSet_Clean(t *testing.T) { + s := NewSet[string](MaxAge(time.Nanosecond)) + s.Set("k") + time.Sleep(time.Millisecond) + s.Clean() + if _, ks := s.TryGet("k"); !ks.IsMiss() { + t.Fatalf("TryGet after Clean on expired: state=%v want Miss", ks) + } +} + +func TestSet_Clear(t *testing.T) { + s := NewSet[string]() + s.Set("a") + s.Set("b") + s.Clear() + if _, ks := s.TryGet("a"); !ks.IsMiss() { + t.Fatalf("after Clear: state=%v want Miss", ks) + } + // Range yields nothing. + var got []string + s.Range(func(k string, _ error) bool { + got = append(got, k) + return true + }) + if len(got) != 0 { + t.Fatalf("Range after Clear: %v, want empty", got) + } +} + +func TestSet_AutoClean(t *testing.T) { + // NewSet must actually start the autoclean goroutine (a regression had + // NewSet building the Cache directly, silently ignoring the option); + // the cleaning behavior itself is covered by the Cache autoclean tests. + s := NewSet[string](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + if s.c.quitClean == nil { + t.Fatal("NewSet with AutoCleanInterval should start the autoclean goroutine") + } + s.StopAutoClean() + // Calling twice is safe (sync.Once on the underlying Cache). + s.StopAutoClean() +} + +func TestItem_AutoClean(t *testing.T) { + i := NewItem[int](MaxAge(time.Millisecond), AutoCleanInterval(time.Millisecond)) + if i.c.quitClean == nil { + t.Fatal("NewItem with AutoCleanInterval should start the autoclean goroutine") + } + defer i.StopAutoClean() + i.Set(1) + i.Clean() // manual Clean is also exposed on Item +} + +func TestSet_ZeroValue(t *testing.T) { + var s Set[string] + err, ks := s.Get("a", func() error { return nil }) + if err != nil || !ks.IsMiss() { + t.Fatalf("zero Set Get: err=%v state=%v", err, ks) + } +} diff --git a/go.mod b/go.mod index da78d2d..39644da 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/twmb/go-cache -go 1.18 +go 1.24 retract v1.1.0 // The newly added API cannot work.