diff --git a/README.md b/README.md index fd3fab601..7f22316d7 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Elastickv is an experimental project undertaking the challenge of creating a dis - **DynamoDB Compatibility Scope**: `CreateTable`/`DeleteTable`/`DescribeTable`/`ListTables`/`PutItem`/`GetItem`/`DeleteItem`/`UpdateItem`/`Query`/`Scan`/`BatchWriteItem`/`TransactWriteItems` are implemented. - **S3 Compatibility Scope**: `ListBuckets`, `CreateBucket`, `HeadBucket`, `DeleteBucket`, `PutObject`, `GetObject`, `HeadObject`, `DeleteObject`, and `ListObjectsV2` (path-style) are implemented. AWS Signature Version 4 authentication with static credentials is supported. The server exposes an S3-compatible HTTP endpoint via `--s3Address`. - **Basic Consistency Behaviors**: Write-after-read checks, leader redirection/forwarding paths, and OCC conflict detection for transactional writes are covered by tests. -- **Hybrid Logical Clock (HLC)**: Transactions are ordered by a 64-bit HLC split into an upper 48-bit physical component (Unix milliseconds) and a lower 16-bit logical counter. The logical half advances in memory with atomic CAS on every `Next()` call — no Raft round-trip per timestamp — so timestamp issuance stays in the nanosecond range. The physical half is bounded by a leader-lease style ceiling: the leader periodically commits a lease entry (`hlcRenewalInterval ≈ 1s`, window `hlcPhysicalWindowMs = 3s`) so that a newly elected leader inherits a safe lower bound and never issues timestamps overlapping the previous leader's window, without blocking per-request on consensus. See `docs/architecture_overview.md` §4 for details. +- **Hybrid Logical Clock (HLC)**: Transactions are ordered by a 64-bit HLC split into an upper 48-bit physical component (Unix milliseconds) and a lower 16-bit logical counter. The logical half advances in memory with atomic CAS on every `Next()` call — no Raft round-trip per timestamp — so timestamp issuance stays in the nanosecond range. The physical half is bounded by a leader-lease style ceiling: the leader periodically commits a lease entry (`hlcRenewalInterval ≈ 2s`, window `hlcPhysicalWindowMs = 20s`). Timestamp safety across leadership changes relies on the new leader observing committed HLC values and on `NextFenced` refusing persistence timestamps after the ceiling expires, without blocking per-request on consensus. See `docs/architecture_overview.md` §4 for details. ## Planned Features - **Dynamic Node Scaling**: Automatic node/range scaling based on load is not yet implemented (current sharding operations are configuration/manual driven). @@ -202,9 +202,10 @@ docker run --rm \ -listen :6479 \ -primary redis.internal:6379 \ -secondary elastickv.internal:6380 \ - -elastickv-pool-size 4 \ - -secondary-write-concurrency 2 \ - -secondary-script-concurrency 1 \ + -elastickv-pool-size 64 \ + -secondary-write-concurrency 32 \ + -secondary-script-concurrency 16 \ + -secondary-blocking-replay-concurrency 32 \ -mode dual-write ``` diff --git a/adapter/redis_bzpopmin_wake_test.go b/adapter/redis_bzpopmin_wake_test.go index 3d1a6eba2..070d84ad0 100644 --- a/adapter/redis_bzpopmin_wake_test.go +++ b/adapter/redis_bzpopmin_wake_test.go @@ -1,15 +1,245 @@ package adapter import ( + "bytes" "context" "runtime" "testing" "time" + "github.com/bootjp/elastickv/store" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" ) +type bzpopminScanRecord struct { + start []byte + limit int +} + +type bzpopminRecordingStore struct { + store.MVCCStore + scans []bzpopminScanRecord +} + +func (s *bzpopminRecordingStore) ScanAt(ctx context.Context, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + s.scans = append(s.scans, bzpopminScanRecord{start: bytes.Clone(start), limit: limit}) + return s.MVCCStore.ScanAt(ctx, start, end, limit, ts) +} + +func TestRedis_BZPopMinCandidateSelection(t *testing.T) { + t.Parallel() + ctx := context.Background() + readTS := uint64(10) + + for _, tc := range []struct { + name string + seed func(t *testing.T, st store.MVCCStore, key []byte) + wantMember string + wantScore float64 + wantWide bool + wantLast bool + wantScans []int + }{ + { + name: "wide score index", + seed: func(t *testing.T, st store.MVCCStore, key []byte) { + seedZSetScoreRowsForTest(t, st, key, readTS, []redisZSetEntry{ + {Member: "later", Score: 2}, + {Member: "first", Score: 1}, + }) + }, + wantMember: "first", + wantScore: 1, + wantWide: true, + wantLast: false, + wantScans: []int{store.MaxDeltaScanLimit + 1, bzpopminScoreScanLimit}, + }, + { + name: "score tie falls back to member ordering", + seed: func(t *testing.T, st store.MVCCStore, key []byte) { + seedZSetScoreRowsForTest(t, st, key, readTS, []redisZSetEntry{ + {Member: "aa", Score: 1}, + {Member: "a", Score: 1}, + }) + }, + wantMember: "a", + wantScore: 1, + wantWide: true, + wantLast: false, + wantScans: []int{store.MaxDeltaScanLimit + 1, bzpopminScoreScanLimit, 1, maxWideScanLimit}, + }, + { + name: "single score entry", + seed: func(t *testing.T, st store.MVCCStore, key []byte) { + seedZSetScoreRowsForTest(t, st, key, readTS, []redisZSetEntry{ + {Member: "only", Score: 3}, + }) + }, + wantMember: "only", + wantScore: 3, + wantWide: true, + wantLast: true, + wantScans: []int{store.MaxDeltaScanLimit + 1, bzpopminScoreScanLimit}, + }, + { + name: "mixed partial score index uses lower member row", + seed: func(t *testing.T, st store.MVCCStore, key []byte) { + seedZSetMemberRowsForBZPopMinTest(t, st, key, readTS, []redisZSetEntry{ + {Member: "low-unindexed", Score: 1}, + {Member: "high-indexed", Score: 5}, + }) + ctx := context.Background() + require.NoError(t, st.PutAt( + ctx, + store.ZSetScoreKey(key, 5, []byte("high-indexed")), + []byte{}, + readTS, + 0, + )) + }, + wantMember: "low-unindexed", + wantScore: 1, + wantWide: true, + wantLast: false, + wantScans: []int{store.MaxDeltaScanLimit + 1, bzpopminScoreScanLimit, 1, maxWideScanLimit}, + }, + { + name: "mixed partial score index keeps indexed candidate non-last", + seed: func(t *testing.T, st store.MVCCStore, key []byte) { + seedZSetMemberRowsForBZPopMinTest(t, st, key, readTS, []redisZSetEntry{ + {Member: "low-indexed", Score: 1}, + {Member: "high-unindexed", Score: 5}, + }) + ctx := context.Background() + require.NoError(t, st.PutAt( + ctx, + store.ZSetScoreKey(key, 1, []byte("low-indexed")), + []byte{}, + readTS, + 0, + )) + }, + wantMember: "low-indexed", + wantScore: 1, + wantWide: true, + wantLast: false, + wantScans: []int{store.MaxDeltaScanLimit + 1, bzpopminScoreScanLimit, 1, maxWideScanLimit}, + }, + { + name: "member only fallback", + seed: func(t *testing.T, st store.MVCCStore, key []byte) { + seedZSetMemberRowsForBZPopMinTest(t, st, key, readTS, []redisZSetEntry{ + {Member: "later", Score: 2}, + {Member: "first", Score: 1}, + }) + }, + wantMember: "first", + wantScore: 1, + wantWide: true, + wantLast: false, + wantScans: []int{store.MaxDeltaScanLimit + 1, bzpopminScoreScanLimit, 1, maxWideScanLimit}, + }, + { + name: "truncated meta still falls back to member rows", + seed: func(t *testing.T, st store.MVCCStore, key []byte) { + seedZSetMemberRowsForBZPopMinTest(t, st, key, readTS, []redisZSetEntry{ + {Member: "later", Score: 2}, + {Member: "first", Score: 1}, + }) + ctx := context.Background() + delta := store.MarshalZSetMetaDelta(store.ZSetMetaDelta{LenDelta: 0}) + for i := uint32(0); i < store.MaxDeltaScanLimit+1; i++ { + require.NoError(t, st.PutAt(ctx, store.ZSetMetaDeltaKey(key, readTS, i), delta, readTS, 0)) + } + }, + wantMember: "first", + wantScore: 1, + wantWide: true, + wantLast: false, + wantScans: []int{store.MaxDeltaScanLimit + 1, bzpopminScoreScanLimit, 1, maxWideScanLimit}, + }, + { + name: "legacy blob fallback", + seed: func(t *testing.T, st store.MVCCStore, key []byte) { + seedLegacyZSetForBZPopMinTest(t, st, key, readTS, []redisZSetEntry{ + {Member: "later", Score: 2}, + {Member: "first", Score: 1}, + }) + }, + wantMember: "first", + wantScore: 1, + wantWide: false, + wantLast: false, + wantScans: []int{store.MaxDeltaScanLimit + 1, bzpopminScoreScanLimit, 1}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + key := []byte("zset-bzpopmin-" + tc.name) + base := store.NewMVCCStore() + tc.seed(t, base, key) + rec := &bzpopminRecordingStore{MVCCStore: base} + server := &RedisServer{store: rec} + + candidate, err := server.bzpopminCandidateAt(ctx, key, readTS) + require.NoError(t, err) + require.NotNil(t, candidate) + require.Equal(t, tc.wantWide, candidate.isWide) + require.Equal(t, tc.wantLast, candidate.isLast) + require.Equal(t, tc.wantMember, candidate.entry.Member) + require.InDelta(t, tc.wantScore, candidate.entry.Score, 1e-9) + require.Len(t, rec.scans, len(tc.wantScans)) + for i, wantLimit := range tc.wantScans { + require.Equal(t, wantLimit, rec.scans[i].limit) + } + require.Equal(t, store.ZSetMetaDeltaScanPrefix(key), rec.scans[0].start) + require.Equal(t, store.ZSetScoreScanPrefix(key), rec.scans[1].start) + }) + } +} + +func seedZSetMemberRowsForBZPopMinTest( + t *testing.T, + st store.MVCCStore, + key []byte, + commitTS uint64, + entries []redisZSetEntry, +) { + t.Helper() + ctx := context.Background() + for _, entry := range entries { + require.NoError(t, st.PutAt( + ctx, + store.ZSetMemberKey(key, []byte(entry.Member)), + store.MarshalZSetScore(entry.Score), + commitTS, + 0, + )) + } + require.NoError(t, st.PutAt( + ctx, + store.ZSetMetaKey(key), + store.MarshalZSetMeta(store.ZSetMeta{Len: int64(len(entries))}), + commitTS, + 0, + )) +} + +func seedLegacyZSetForBZPopMinTest( + t *testing.T, + st store.MVCCStore, + key []byte, + commitTS uint64, + entries []redisZSetEntry, +) { + t.Helper() + ctx := context.Background() + raw, err := marshalZSetValue(redisZSetValue{Entries: entries}) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, redisZSetKey(key), raw, commitTS, 0)) +} + // TestRedis_BZPopMinWakesOnZAdd verifies the event-driven wake path: // an in-process ZADD on the leader's redis adapter must wake a // BZPOPMIN waiter on the same node so the reader returns the new diff --git a/adapter/redis_peer_limiter.go b/adapter/redis_peer_limiter.go index 88a3077fa..155590319 100644 --- a/adapter/redis_peer_limiter.go +++ b/adapter/redis_peer_limiter.go @@ -6,13 +6,14 @@ import ( "strconv" "strings" "sync" + + "github.com/bootjp/elastickv/internal/redislimits" ) const ( redisPerPeerLimitEnv = "ELASTICKV_REDIS_PER_PEER_CONNECTIONS" - defaultRedisProxyPoolPeerCap = 64 - defaultRedisDedicatedPeerHeadroom = 64 - defaultRedisPerPeerConnectionCap = defaultRedisProxyPoolPeerCap + defaultRedisDedicatedPeerHeadroom + defaultRedisDedicatedPeerHeadroom = redislimits.DefaultElasticKVRedisConnections + defaultRedisPerPeerConnectionCap = redislimits.DefaultElasticKVRedisConnections + defaultRedisDedicatedPeerHeadroom redisPeerLimitError = "ERR max connections per client exceeded" unknownRedisPeer = "unknown" ) diff --git a/adapter/redis_peer_limiter_test.go b/adapter/redis_peer_limiter_test.go index ddd3b608e..e8f197ff5 100644 --- a/adapter/redis_peer_limiter_test.go +++ b/adapter/redis_peer_limiter_test.go @@ -3,7 +3,7 @@ package adapter import ( "testing" - "github.com/bootjp/elastickv/proxy" + "github.com/bootjp/elastickv/internal/redislimits" "github.com/stretchr/testify/require" ) @@ -13,9 +13,12 @@ const ( func TestRedisPeerLimiterDefaultMatchesProxyPool(t *testing.T) { t.Setenv(redisPerPeerLimitEnv, "") - limiter := newDefaultRedisPeerLimiter() - require.NotNil(t, limiter) - require.Equal(t, proxy.DefaultElasticKVBackendOptions().PoolSize+defaultRedisDedicatedPeerHeadroom, limiter.limit) + server := NewRedisServer(nil, "", nil, nil, nil, nil) + + require.NotNil(t, server.peerLimiter) + require.Equal(t, defaultRedisPerPeerConnectionCap, server.peerLimiter.limit) + require.Equal(t, redislimits.DefaultElasticKVRedisConnections+defaultRedisDedicatedPeerHeadroom, server.peerLimiter.limit) + require.Greater(t, server.peerLimiter.limit, redislimits.DefaultElasticKVRedisConnections) } func TestRedisPeerLimiterRejectsAndReleases(t *testing.T) { diff --git a/adapter/redis_zset_cmds.go b/adapter/redis_zset_cmds.go index 1c4aca8b4..8413985ef 100644 --- a/adapter/redis_zset_cmds.go +++ b/adapter/redis_zset_cmds.go @@ -27,7 +27,17 @@ type bzpopminResult struct { entry redisZSetEntry } -const zsetOpsPerEntry = 2 +type bzpopminCandidate struct { + entry redisZSetEntry + remaining []redisZSetEntry + isWide bool + isLast bool +} + +const ( + zsetOpsPerEntry = 2 + bzpopminScoreScanLimit = 2 +) // buildZSetLegacyMigrationElems returns ops that atomically migrate a legacy // !redis|zset| blob to wide-column !zs|mem| + !zs|scr| keys. Returns nil if no @@ -1128,44 +1138,174 @@ func (r *RedisServer) tryBZPopMinWithMode(key []byte, fast bool) (*bzpopminResul if typ != redisTypeZSet { return wrongTypeError() } - value, _, err := r.loadZSetAt(ctx, key, readTS) + candidate, err := r.bzpopminCandidateAt(ctx, key, readTS) if err != nil { return err } - if len(value.Entries) == 0 { + if candidate == nil { result = nil return nil } - popped := value.Entries[0] - remaining := append([]redisZSetEntry(nil), value.Entries[1:]...) - - // Detect wide-column storage. - memberPrefix := store.ZSetMemberScanPrefix(key) - memberEnd := store.PrefixScanEnd(memberPrefix) - probeKVs, probeErr := r.store.ScanAt(ctx, memberPrefix, memberEnd, 1, readTS) - if probeErr != nil { - return cockerrors.WithStack(probeErr) - } - isWide := len(probeKVs) > 0 - - if err := r.persistBZPopMinResult(ctx, key, readTS, popped, remaining, isWide); err != nil { + if err := r.persistBZPopMinResult(ctx, key, readTS, candidate); err != nil { return err } - result = &bzpopminResult{key: key, entry: popped} + result = &bzpopminResult{key: key, entry: candidate.entry} return nil }) return result, err } -func (r *RedisServer) persistBZPopMinResult(ctx context.Context, key []byte, readTS uint64, popped redisZSetEntry, remaining []redisZSetEntry, isWide bool) error { - if len(remaining) == 0 { +func (r *RedisServer) bzpopminCandidateAt(ctx context.Context, key []byte, readTS uint64) (*bzpopminCandidate, error) { + scoreCandidate, scoreIndexComplete, err := r.bzpopminWideScoreCandidateAt(ctx, key, readTS) + if err != nil { + return nil, err + } + if scoreIndexComplete { + return scoreCandidate, nil + } + memberCandidate, err := r.bzpopminMemberOnlyCandidateAt(ctx, key, readTS) + if err != nil { + return nil, err + } + if scoreCandidate != nil && memberCandidate != nil { + if zsetEntryLess(scoreCandidate.entry, memberCandidate.entry) { + return scoreCandidate, nil + } + return memberCandidate, nil + } + if scoreCandidate != nil { + return scoreCandidate, nil + } + if memberCandidate != nil { + return memberCandidate, nil + } + return r.bzpopminLegacyCandidateAt(ctx, key, readTS) +} + +func zsetEntryLess(left, right redisZSetEntry) bool { + if left.Score != right.Score { + return left.Score < right.Score + } + return left.Member < right.Member +} + +func (r *RedisServer) bzpopminWideScoreCandidateAt(ctx context.Context, key []byte, readTS uint64) (*bzpopminCandidate, bool, error) { + metaLen, metaExists, err := r.resolveZSetMeta(ctx, key, readTS) + if err != nil { + if !errors.Is(err, ErrDeltaScanTruncated) { + return nil, false, err + } + metaLen = 0 + metaExists = false + } + scorePrefix := store.ZSetScoreScanPrefix(key) + scoreEnd := store.PrefixScanEnd(scorePrefix) + scoreKVs, err := r.store.ScanAt(ctx, scorePrefix, scoreEnd, bzpopminScoreScanLimit, readTS) + if err != nil { + return nil, false, cockerrors.WithStack(err) + } + candidate, hasTie := bzpopminScoreCandidateFromKVs(key, scoreKVs) + if hasTie { + return nil, false, nil + } + if candidate != nil { + scoreIndexComplete := metaExists && metaLen > 0 && int64(len(scoreKVs)) == metaLen + candidate.isLast = scoreIndexComplete && metaLen == 1 + return candidate, scoreIndexComplete, nil + } + return nil, metaExists && metaLen == 0, nil +} + +func bzpopminScoreCandidateFromKVs(key []byte, scoreKVs []*store.KVPair) (*bzpopminCandidate, bool) { + var firstScore float64 + var candidate *bzpopminCandidate + for _, kv := range scoreKVs { + score, member, ok := store.ExtractZSetScoreAndMember(kv.Key, key) + if !ok { + continue + } + if candidate == nil { + firstScore = score + candidate = &bzpopminCandidate{ + entry: redisZSetEntry{Member: string(member), Score: score}, + isWide: true, + } + continue + } + if zsetScoresEqual(firstScore, score) { + return nil, true + } + break + } + return candidate, false +} + +func zsetScoresEqual(left, right float64) bool { + if left == 0 && right == 0 { + return true + } + return math.Float64bits(left) == math.Float64bits(right) +} + +func (r *RedisServer) bzpopminMemberOnlyCandidateAt(ctx context.Context, key []byte, readTS uint64) (*bzpopminCandidate, error) { + memberPrefix := store.ZSetMemberScanPrefix(key) + memberEnd := store.PrefixScanEnd(memberPrefix) + memberKVs, err := r.store.ScanAt(ctx, memberPrefix, memberEnd, 1, readTS) + if err != nil { + return nil, cockerrors.WithStack(err) + } + if len(memberKVs) > 0 { + value, loadErr := r.loadZSetMembersAt(ctx, key, readTS) + if loadErr != nil { + return nil, loadErr + } + if len(value.Entries) == 0 { + return nil, nil + } + return &bzpopminCandidate{ + entry: value.Entries[0], + remaining: append([]redisZSetEntry(nil), value.Entries[1:]...), + isWide: true, + isLast: len(value.Entries) == 1, + }, nil + } + return nil, nil +} + +func (r *RedisServer) bzpopminLegacyCandidateAt(ctx context.Context, key []byte, readTS uint64) (*bzpopminCandidate, error) { + raw, err := r.store.GetAt(ctx, redisZSetKey(key), readTS) + if err != nil { + if cockerrors.Is(err, store.ErrKeyNotFound) { + return nil, nil + } + return nil, cockerrors.WithStack(err) + } + value, err := unmarshalZSetValue(raw) + if err != nil { + return nil, err + } + if len(value.Entries) == 0 { + return nil, nil + } + return &bzpopminCandidate{ + entry: value.Entries[0], + remaining: append([]redisZSetEntry(nil), value.Entries[1:]...), + isLast: len(value.Entries) == 1, + }, nil +} + +func (r *RedisServer) persistBZPopMinResult(ctx context.Context, key []byte, readTS uint64, candidate *bzpopminCandidate) error { + if candidate == nil { + return nil + } + if candidate.isLast { elems, _, err := r.deleteLogicalKeyElems(ctx, key, readTS) if err != nil { return err } return r.dispatchElems(ctx, true, readTS, elems) } - if isWide { + if candidate.isWide { // Wide-column: delete the popped member key + score index, emit delta -1. startTS := normalizeStartTS(readTS) commitTS, err := r.nextCommitTSAfter(ctx, startTS, "persistBZPopMinResult: allocate commitTS") @@ -1174,8 +1314,8 @@ func (r *RedisServer) persistBZPopMinResult(ctx context.Context, key []byte, rea } deltaVal := store.MarshalZSetMetaDelta(store.ZSetMetaDelta{LenDelta: -1}) elems := []*kv.Elem[kv.OP]{ - {Op: kv.Del, Key: store.ZSetMemberKey(key, []byte(popped.Member))}, - {Op: kv.Del, Key: store.ZSetScoreKey(key, popped.Score, []byte(popped.Member))}, + {Op: kv.Del, Key: store.ZSetMemberKey(key, []byte(candidate.entry.Member))}, + {Op: kv.Del, Key: store.ZSetScoreKey(key, candidate.entry.Score, []byte(candidate.entry.Member))}, {Op: kv.Put, Key: store.ZSetMetaDeltaKey(key, commitTS, 0), Value: deltaVal}, redisTxnWideZSetFenceElem(key), } @@ -1189,7 +1329,7 @@ func (r *RedisServer) persistBZPopMinResult(ctx context.Context, key []byte, rea return cockerrors.WithStack(dispatchErr) } // Legacy blob: write back all remaining entries. - payload, err := marshalZSetValue(redisZSetValue{Entries: remaining}) + payload, err := marshalZSetValue(redisZSetValue{Entries: candidate.remaining}) if err != nil { return err } diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 7188637f9..b67b3922c 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -75,7 +75,7 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { elasticKVPoolSize: 64, wantWriteConcurrency: 32, wantScriptConcurrency: 16, - wantBlockingConcurrency: 20, + wantBlockingConcurrency: 32, }, { name: "shadow mode derives from ElasticKV pool", @@ -93,17 +93,17 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { elasticKVPoolSize: 4, wantWriteConcurrency: 64, wantScriptConcurrency: 32, - wantBlockingConcurrency: 20, + wantBlockingConcurrency: 64, }, { - name: "large remaining pool caps blocking replay", + name: "large remaining pool caps blocking replay at production default", mode: proxy.ModeDualWrite, primaryPoolSize: 128, elasticKVPoolSize: 144, writeConcurrency: 80, wantWriteConcurrency: 80, wantScriptConcurrency: 40, - wantBlockingConcurrency: 20, + wantBlockingConcurrency: 64, }, { name: "explicit write keeps derived script", diff --git a/docs/architecture_overview.md b/docs/architecture_overview.md index 796d3d7fe..ba4bc1ea3 100644 --- a/docs/architecture_overview.md +++ b/docs/architecture_overview.md @@ -170,7 +170,7 @@ sequenceDiagram participant H as "HLC (all nodes)" participant Tx as "Txn / MVCC read-write path" - loop "every hlcRenewalInterval (<3s)" + loop "every hlcRenewalInterval (=2s; <20s)" L->>RG: "Propose HLC lease (now + hlcPhysicalWindowMs)" RG-->>F: "Apply HLC lease entry" F->>H: "SetPhysicalCeiling(ms)" diff --git a/docs/design/2026_04_24_implemented_workload_isolation.md b/docs/design/2026_04_24_implemented_workload_isolation.md index 8ee69b3ee..c20f5f8cc 100644 --- a/docs/design/2026_04_24_implemented_workload_isolation.md +++ b/docs/design/2026_04_24_implemented_workload_isolation.md @@ -24,8 +24,9 @@ Implementation status: gated `EVAL`/`EVALSHA` path. - Shipped: Layer 3 per-peer Redis connection admission in `adapter/redis_peer_limiter.go`, wired through `RedisServer.Run` accept and - close hooks. Default cap is 8 per peer IP and is configurable via - `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` / + close hooks. Default cap is 128 per peer IP: 64 ElasticKV command-pool slots + plus 64 dedicated headroom for Pub/Sub and detached sockets. It is + configurable via `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` / `WithRedisPerPeerConnectionLimit`. The redis-proxy deployment can raise the cap explicitly on ElasticKV nodes before increasing the proxy's ElasticKV pool size. Redis leader-proxy clients use a small explicit go-redis pool @@ -376,7 +377,7 @@ one check per accept, not per command. ### Recommended v1 shape -**Per-peer-IP connection cap, default `N=8`, env-configurable, +**Per-peer-IP connection cap, default `N=128`, env-configurable, enforced at accept.** On reject, accept the TCP connection, write a `-ERR max connections per client exceeded` RESP error, then close — so the client sees a protocol-level message instead of a bare diff --git a/docs/design/2026_05_28_implemented_tla_safety_spec.md b/docs/design/2026_05_28_implemented_tla_safety_spec.md index cd94bb5d0..ea6d6aa81 100644 --- a/docs/design/2026_05_28_implemented_tla_safety_spec.md +++ b/docs/design/2026_05_28_implemented_tla_safety_spec.md @@ -199,8 +199,8 @@ cannot check them as state invariants. independently reach `ceiling + 1` before a fresh ceiling is renewed, the new leader's first `Next()` can tie or undercut the old leader's last commit. Bounding inter-node skew to less than - one ceiling window (< 3s with the current `hlcPhysicalWindowMs = - 3000ms`) keeps the window wide enough that the new leader cannot + one ceiling window (< 20s with the current `hlcPhysicalWindowMs = + 20000ms`) keeps the window wide enough that the new leader cannot independently reach the overflow value before a renewal applies. Should be surfaced in operator docs as a cluster prerequisite. - **(ii) Logical-counter handoff.** The 16-bit logical half of the HLC @@ -244,33 +244,28 @@ cannot check them as state invariants. - **(iii) Commit-time ceiling fencing.** Bounded skew (i) and logical handoff (ii) are not enough on their own: a leader that misses `RunHLCLeaseRenewal` for longer than `hlcPhysicalWindowMs` (e.g. - partitioned from the quorum that owns the ceiling group — in the - `ShardedCoordinator` the ceiling group is always - `c.defaultGroup`, per `kv/sharded_coordinator.go`, so the failure - scenario is: a node that is leader of a non-default shard but - partitioned from the default group's quorum) but remains a Raft - leader will continue advancing its wall clock past the last - committed ceiling and can still issue persistence timestamps. A - subsequent leader that has only restored the same stale ceiling - and has a lagging wall clock can then issue a strictly smaller - timestamp, violating HLC-4. The spec therefore requires every - `Next()` that backs a persistence ts to be gated on - `wall_now < physicalCeiling[n]` (the same `physicalCeiling[n]` - variable as §3, matching `kv/hlc.go`'s field); a leader whose - ceiling has expired must either re-apply a fresh ceiling via - Raft before committing, or refuse to commit (fail-closed). The - current implementation in `kv/sharded_coordinator.go` - `RunHLCLeaseRenewal` only logs renewal failures and dispatch - paths call `HLC.Next()` without checking ceiling freshness — this - is the second design gap the spec is meant to surface. As with - (ii), if the gap is real, TLC will produce a counterexample for - HLC-4 under a model that lets renewals fail. **Availability - note.** The fail-closed behaviour means a leader partitioned - from the default group's quorum for longer than - `hlcPhysicalWindowMs` cannot serve any persistence timestamp — - every client commit is rejected until renewal succeeds. This is - a CP, not AP, trade-off and operators must size - `hlcPhysicalWindowMs` (currently 3s) relative to expected + partitioned from the quorum for a shard group whose timestamps it + renews) but remains a Raft leader can advance wall clock past the + last committed ceiling. A subsequent leader that has only restored + the same stale ceiling and has a lagging wall clock can then issue a + strictly smaller timestamp, violating HLC-4, unless persistence-grade + allocation fails closed. The spec therefore requires every timestamp + that backs persistence to be gated on `wall_now < physicalCeiling[n]` + (the same `physicalCeiling[n]` variable as §3, matching `kv/hlc.go`'s + field). The current implementation renews HLC leases per led shard + group through `ShardedCoordinator.RunHLCLeaseRenewal`, and persistence + allocation uses `NextFenced`; if the ceiling is expired or the + logical half is exhausted before a fresh renewal applies, the + committing path refuses with `ErrCeilingExpired` and may use + `RecoverHLCLease` to synchronously propose a fresh ceiling before + retrying. As with (ii), TLC must still model failed renewals because + the availability outcome is a client-visible rejection, not silent + timestamp issuance. **Availability note.** The fail-closed behaviour + means a shard leader partitioned from that shard group's quorum for + longer than `hlcPhysicalWindowMs` cannot serve any persistence + timestamp — every client commit on that group is rejected until + renewal succeeds. This is a CP, not AP, trade-off and operators must size + `hlcPhysicalWindowMs` (currently 20s) relative to expected partition duration; see §9 risk 7. ### 5.2 OCC @@ -674,21 +669,20 @@ does not keep this document in `partial`. specific version and checksum. 7. **Fail-closed availability under partition.** HLC-4 precondition - (iii) makes the ceiling-fence behaviour normative: a leader - partitioned from the default group's quorum for longer than - `hlcPhysicalWindowMs` (currently 3s) cannot serve any persistence - timestamp, so client commits are rejected until renewal succeeds. - This is a CP, not AP, trade-off and is a stricter regime than the - current implementation (which silently keeps issuing). Mitigation: - the M1 spec encodes the fence as an **action guard** on - `IssueTimestamp` (`ENABLED ⇔ wallNow[n] < physicalCeiling[n]`, per - §8.1 — `ASSUME` is only valid for constant predicates and does not - apply here); the follow-up implementation PR that adds the fence - to `HLC.Next()` / - `ShardedCoordinator.RunHLCLeaseRenewal` must (a) surface the new - error in operator-visible metrics and (b) document the - `hlcPhysicalWindowMs` tuning guidance — operators sizing the - window must trade it against expected partition duration. + (iii) makes the ceiling-fence behaviour normative: a shard leader + partitioned from that shard group's quorum for longer than + `hlcPhysicalWindowMs` (currently 20s) cannot serve persistence + timestamps for that group. Persistence allocation goes through + `NextFenced`; when renewal has not advanced the ceiling in time, + client commits fail closed with `ErrCeilingExpired` unless + `RecoverHLCLease` can synchronously apply a fresh ceiling. This is + a CP, not AP, trade-off. Mitigation: the M1 spec encodes the fence + as an **action guard** on `IssueTimestamp` + (`ENABLED ⇔ wallNow[n] < physicalCeiling[n]`, per §8.1 — `ASSUME` + is only valid for constant predicates and does not apply here); + implementation surfaces the rejection via `NextFencedRejections` + and operator docs must keep `hlcPhysicalWindowMs` tuning guidance + tied to expected partition duration and renewal latency. --- diff --git a/docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md b/docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md index 2317fb9db..c8f78819b 100644 --- a/docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md +++ b/docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md @@ -643,7 +643,7 @@ entries individually. Round-3 of this doc claimed the resulting full-restore fallback was "safe and rare." Codex correctly pointed out it is neither: `RunHLCLeaseRenewal` (`kv/coordinator.go:650`) proposes a lease every -`hlcRenewalInterval = 1 * time.Second` while the local node is leader, +`hlcRenewalInterval = 2 * time.Second` while the local node is leader, so even an active cluster will routinely accumulate a tail of lease entries between any two data writes. When the next snapshot is persisted at index `X`, the gap between the last data-Apply index `Y` @@ -945,9 +945,9 @@ Each of B2–B3 ships behind tests: An idle-cluster integration test runs a 3-node cluster with `ELASTICKV_RAFT_SNAPSHOT_COUNT=10` (overriding the default `defaultSnapshotEvery = 10000` at `engine.go:93` so the scenario is - tractable — at default + `hlcRenewalInterval = 1 s` an idle period - would need ≥ 20 000 s). With the override, the test issues no data - writes for `2 × 10 × hlcRenewalInterval = 20 s`, takes a snapshot, + tractable — at default + `hlcRenewalInterval = 2 s` an idle period + would need ≥ 40 000 s). With the override, the test issues no data + writes for `2 × 10 × hlcRenewalInterval = 40 s`, takes a snapshot, restarts a node, and asserts the skip fires — proving the codex round-3 P2 scenario is closed end-to-end through the `e.persistLocalSnapshotPayload` hook added in round-5. @@ -1043,7 +1043,7 @@ Codex's round-3 review (P2 at `:438`) pointed out the actual production cadence: - `RunHLCLeaseRenewal` (`kv/coordinator.go:650`) ticks at - `hlcRenewalInterval = 1 * time.Second` while the local node is + `hlcRenewalInterval = 2 * time.Second` while the local node is leader. - `applyHLCLease` is memory-only; `metaAppliedIndex` does not advance on lease apply. diff --git a/docs/design/2026_06_11_implemented_leader_balance_scheduler.md b/docs/design/2026_06_11_implemented_leader_balance_scheduler.md index 36305731a..7b8c418d7 100644 --- a/docs/design/2026_06_11_implemented_leader_balance_scheduler.md +++ b/docs/design/2026_06_11_implemented_leader_balance_scheduler.md @@ -189,7 +189,7 @@ A group is **eligible** for a transfer this cycle only if **all** of the followi - **SQS leadership refusal.** A node lacking the `htfifo` capability refuses leadership of any group hosting a partitioned FIFO queue (`main_sqs_leadership_refusal.go:69-121`, wired from `--sqsFifoPartitionMap` via `partitionedGroupSet`, `main_sqs_leadership_refusal.go:136-156`, `shard_config.go:174-196`). If the balancer transferred such a group onto a refusing node, that node would immediately transfer it away again — a guaranteed ping-pong. The balancer must therefore **know which (group, node) pairs are refusing** and exclude them as transfer targets. **How it knows:** the same configuration that drives the refusal hook — the partitioned-group set (`partitionedGroupSet`) plus each node's advertised `htfifo` capability from the PR2 capability report — is the source of truth. For any group in the partitioned-FIFO set, the balancer includes only candidate targets that currently advertise `htfifo`; if capability is missing/stale for any candidate, it fails closed by excluding that target, and if no htfifo-capable target remains the group is skipped with `sqs_refused` / `no_eligible_move`. - **Pinning flag ships with the first transfer PR.** `--leaderBalancePinGroups` (and/or per-group pin) is part of PR2, so an operator can pin a group's leadership (e.g. to a node with special hardware) and keep the live transfer scheduler off it from day one. - **Default group is balanceable, with a called-out blip.** The default group is *not* excluded — leaving it pinned to one node defeats half the point (that node also carries HLC/catalog work). But moving the default group's leadership has two transient effects the operator must understand: - - **HLC ceiling renewal** restarts on the new leader. The ceiling is proposed every `hlcRenewalInterval` (1 s) with a `hlcPhysicalWindowMs` (3 s) window (`kv/coordinator.go:37-46`, `:644-669`); because the window (3 s) exceeds the renewal interval (1 s) and a newly elected leader clamps `Next()` to `max(wall, ceiling)`, a clean transfer does **not** let the new leader issue a timestamp inside the old leader's window — the safety invariant holds across a transfer exactly as it does across a natural election. + - **HLC ceiling renewal** restarts on the new leader. The ceiling is proposed every `hlcRenewalInterval` (2 s) with a `hlcPhysicalWindowMs` (20 s) window (`kv/coordinator.go:37-51`, `:644-669`); with the logical-counter handoff that observes the store's last committed HLC and the `NextFenced` fail-closed ceiling fence active, a clean transfer does **not** let the new leader issue a timestamp inside the old leader's window — the safety invariant holds across a transfer exactly as it does across a natural election. - **Lease reads:** a transfer invalidates the lease on the old leader (`RegisterLeaderLossCallback → lease.invalidate`, `kv/coordinator.go:131`, `kv/sharded_coordinator.go:584`), so the new leader's *first* read takes the slow `LinearizableRead` path (one ReadIndex round-trip) before its lease warms again — a single read-latency blip, not a correctness issue. This is identical to the blip on any natural election. The cooldowns keep these blips rare. - **Recommendation (resolving OQ-3): balance the default group LAST — gated by "no eligible non-default objective-improving move exists this cycle", not by other groups' source counts (resolves codex round-7 P2).** The default group is only admitted to the eligible-source set on a **second pass** of the §3.3 candidate iteration, executed only when the first pass — restricted to non-default groups — found no `(source, group, target)` tuple that improves the objective tuple. Concretely each cycle runs: 1. **First pass — non-default only.** Run the §3.3 descending-source-count iteration over candidate source nodes, considering only groups whose `group_id != defaultGroupID`. If any source yields an eligible (group, target) pair (§3.5 conf-change / in-flight / catch-up / liveness / not policy-pinned, all unchanged), issue exactly that transfer and stop (single transfer per cycle is preserved). @@ -245,7 +245,7 @@ Each PR carries the five-lens self-review and is gated by its tests + `make lint | **Concurrency / distributed** | Scheduler goroutine vs. default-group leadership flip (startup grace + cluster-scoped kill switch, §3.1/§3.6); follower-initiated transfer is impossible and must be forwarded (§3.4). **Partial observation:** any no-leader group disables transfers for the cycle, so the policy never acts on a count map missing leaders. **Forward-path TOCTOU:** if the leader moved between observation and dial, remote `handleTransferLeadership` rejects with `errLeadershipTransferNotLeader` and the balancer records a skip. **Target catch-up AND liveness are validated by the executing group leader** via gated `TransferLeadershipToServerIfEligible`, which checks `Match >= saturating_sub(LastLogIndex, maxLag)` + `Progress.RecentActive` before `rawNode.TransferLeader`; the balancer only sends an ordered target preference list. **Conf-change race is enforced on the executing leader from raft-level pending-conf state**, not request-local `pendingConfigs`; the guard must survive canceled requests and leadership changes, and existing SQS/system callers retry its transient error. **In-flight-transfer race is also enforced on the executing leader** via an unconditional different-target `LeadTransferee` guard before `rawNode.TransferLeader`, so a forwarded balancer request cannot silently cancel an operator/system transfer; same-target duplicates are idempotent. Run `go test -race ./kv/... ./internal/raftengine/...` and the matching Jepsen suite. | | **Security** | The forward path uses the `RaftAdmin` gRPC service, which is **unauthenticated** today (`--adminTokenFile` gates only `/Admin/`, `adapter/admin_grpc.go:484`,`:498`). Confirm the `RaftAdmin` listener is on a trusted boundary (the inter-node Raft transport boundary); if/when `RaftAdmin` is put behind a token/mTLS (OQ-11), the internal node-to-node RaftAdmin client must present the credential (§3.4). **Mixed-version forward:** every forwarded `gated = true` request is preceded by a fresh/current `transfer_gate` capability check plus the rollout-rule precondition; an old leader that would drop the fields is skipped rather than trusted. | | **Performance** | Observation is local `Status()`/`Leader()` reads on a 30 s ticker — no Raft round-trips, no per-`Next()` HLC consensus, no Pebble reads (§3.2). One transfer per cycle bounds churn. Metric cardinality is fixed-enum / per-node only (§3.6). No hot-path code path is touched. | -| **Data consistency** | A transfer perturbs the default group's HLC renewal and lease reads transiently but preserves the HLC physical-ceiling invariant (a clean transfer is no different from a natural election; the 3 s window > 1 s renewal guarantees no in-window reissue, §3.5). Lease-read freshness is preserved — the new leader falls back to `LinearizableRead` until its lease re-warms. No route-catalog mutation occurs (the balancer never writes the catalog). | +| **Data consistency** | A transfer perturbs the default group's HLC renewal and lease reads transiently but preserves the HLC physical-ceiling invariant (a clean transfer is no different from a natural election; the 20 s window leaves margin above the 2 s renewal cadence and 5 s proposal timeout, §3.5). Lease-read freshness is preserved — the new leader falls back to `LinearizableRead` until its lease re-warms. No route-catalog mutation occurs (the balancer never writes the catalog). | | **Test coverage** | Pure-policy table tests + `rapid` invariants + 3-node convergence/anti-ping-pong integration + partial-observation skip + cluster-scoped kill-switch failover + leader-change reset; Jepsen "no new anomalies with balancer on" run recorded. SQS-refusal retry, pinning exclusion, ordered target fall-through, and same-target transfer idempotency are PR2 tests because PR2 is the first transfer-issuing milestone. | ## 7. Future work (out of scope for v1) diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index 716b6bdd7..8cddf007c 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -84,12 +84,13 @@ Breakage points: references anywhere in `kv/`, `distribution/`, or `internal/raftengine/etcd/`. Everything is single-region etcd/raft. -HLC is `kv/hlc.go` + `kv/coordinator.go`: physical ceiling is -Raft-agreed via `ProposeHLCLease`, `hlcPhysicalWindowMs = 3 s`, -renewed every `hlcRenewalInterval = 1 s` from the **default-group -leader only** (`ShardedCoordinator.RunHLCLeaseRenewal`). `NextFenced` -fails closed with `ErrCeilingExpired` once `wall_now >= ceiling`. -Wall clock is `time.Now().UnixMilli()` — assumes NTP-synced hosts. +HLC is `kv/hlc.go` + `kv/coordinator.go` + +`kv/sharded_coordinator.go`: physical ceiling is Raft-agreed via +`ProposeHLCLease`, `hlcPhysicalWindowMs = 20 s`, renewed every +`hlcRenewalInterval = 2 s` by each led shard group +(`ShardedCoordinator.RunHLCLeaseRenewal`). `NextFenced` fails closed +with `ErrCeilingExpired` once `wall_now >= ceiling`. Wall clock is +`time.Now().UnixMilli()` — assumes NTP-synced hosts. Raft engine (`internal/raftengine/etcd/engine.go`): `defaultTickInterval = 10 ms`, `defaultHeartbeatTick = 10` (100 ms), @@ -98,10 +99,11 @@ LAN-tuned; would spuriously election-time-out cross-WAN. Multi-region blockers: -- HLC ceiling is **single-point** — default-group leader proposes - every 1 s; cross-region default-group leadership means every - shard's persistence-grade ts depends on a cross-WAN propose. 1 s - WAN RTT + 1 s renewal interval ≈ no margin against the 3 s window. +- HLC ceiling renewal is **per Raft group** — each shard group leader + proposes every 2 s, so cross-region leadership for that group makes + its persistence-grade ts depend on cross-WAN quorum progress. 1 s WAN + RTT is now less likely to expire the 20 s window immediately, but + every renewal still depends on that group's cross-WAN quorum progress. - 100 ms heartbeat / 1 s election timeout cannot run cross-DC. - No TrueTime/clockbound integration. - Cross-shard txns are blocked (`ErrCrossShardTransactionNotSupported`), @@ -316,10 +318,10 @@ whole cluster; cross-WAN that is a 1 s RTT cliff. Options surveyed: service.** Operationally heavy (needs reliable atomic-clock reference per DC); rejected as v1. - **Option C — Stretched single ceiling with relaxed window.** - Increase `hlcPhysicalWindowMs` to 30 s; renewal still cross-WAN - but the failure mode is "slow ts allocation," not "no ts - allocation." Useful as a fallback when option A is partially - shipped. + `hlcPhysicalWindowMs = 20 s` is now the local-resilience baseline. + It reduces accidental ceiling expiry under load, but renewal still + depends on each route's shard group quorum and is not a complete + cross-region design by itself. V1 = option A. Carries the existing M2 hotspot-split monotone-merge contract over the region boundary. diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index f9027487b..3001885e3 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -35,11 +35,13 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-secondary-db` | `0` | Secondary Redis DB number | | `-secondary-password` | (empty) | Secondary Redis password | | `-primary-pool-size` | `128` | Primary Redis backend connection pool size | -| `-elastickv-pool-size` | `64` | ElasticKV backend command pool size; keep the server-side `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` above this to leave room for dedicated PubSub connections | +| `-elastickv-pool-size` | `64` | ElasticKV backend connection pool size; keep this at or below `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` on the cluster | | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` | | `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` | +| `-secondary-blocking-replay-concurrency` | `0` | Mutating blocking-command replay limit. `0` uses the remaining secondary backend pool capacity after normal writes, capped at `64` | | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-secondary-script-queue-size` | `0` | Bounded queue for secondary Lua-script writes. `0` derives `64 * concurrency`, clamped to `64..8192` | +| `-secondary-blocking-replay-queue-size` | `0` | Bounded queue for mutating blocking-command replays. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-mode` | `dual-write` | Proxy mode (see below) | | `-secondary-timeout` | `5s` | End-to-end secondary write deadline, including queue wait | | `-shadow-timeout` | `3s` | Shadow read timeout | @@ -97,6 +99,7 @@ docker run --rm \ -elastickv-pool-size 64 \ -secondary-write-concurrency 32 \ -secondary-script-concurrency 16 \ + -secondary-blocking-replay-concurrency 32 \ -mode dual-write-shadow \ -secondary-timeout 5s \ -shadow-timeout 3s \ @@ -121,6 +124,7 @@ services: - -elastickv-pool-size=64 - -secondary-write-concurrency=32 - -secondary-script-concurrency=16 + - -secondary-blocking-replay-concurrency=32 - -mode=dual-write-shadow - -metrics=:9191 depends_on: @@ -370,7 +374,7 @@ Available at `/metrics` on the address specified by `-metrics`. | `proxy_divergences_total` | Counter | Shadow read mismatches (labels: command, kind) | | `proxy_migration_gap_total` | Counter | Expected mismatches from incomplete migration (labels: command) | | `proxy_async_drops_total` | Counter | Async operations dropped due to backpressure | -| `proxy_async_drops_by_queue_total{queue,reason}` | Counter | Drops split by `write`/`script`/`shadow` and `queue_full`/`expired` | +| `proxy_async_drops_by_queue_total{queue,reason}` | Counter | Drops split by `write`/`script`/`blocking`/`shadow` and `queue_full`/`expired` | | `proxy_async_queue_depth{queue}` | Gauge | Async operations waiting for worker capacity | | `proxy_async_queue_capacity{queue}` | Gauge | Configured queue capacity | | `proxy_async_queue_delay_seconds{queue}` | Histogram | Time spent waiting for worker capacity | @@ -419,6 +423,7 @@ groups: | Read timeout | 3s | Backend read timeout | | Write timeout | 3s | Backend write timeout | | Async write concurrency fallback | 4096 | Package fallback; the command derives a lower limit from backend pool size | +| Async blocking replay concurrency cap | 64 | Command-derived maximum for mutating blocking-command replays after normal writes reserve pool capacity | | Shadow read goroutine limit | 1024 | Max concurrent shadow comparisons | | PubSub compare window | 2s | Message matching window | | PubSub sweep interval | 500ms | Expired message scan interval | @@ -437,7 +442,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. ## Troubleshooting ### Secondary writes are falling behind -- Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. +- Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` by queue before increasing concurrency. A sustained `blocking` queue means BZPOP/XREAD replay work is lagging behind normal writes. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. - Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` above `-elastickv-pool-size`; PubSub and shadow PubSub use dedicated connections outside the command pool and detached PubSub sockets may remain counted until cleanup. Keep `-secondary-write-concurrency` at or below the pool size. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. diff --git a/internal/redislimits/limits.go b/internal/redislimits/limits.go new file mode 100644 index 000000000..a0c82d55c --- /dev/null +++ b/internal/redislimits/limits.go @@ -0,0 +1,5 @@ +package redislimits + +// DefaultElasticKVRedisConnections is the shared default connection budget +// between the Redis proxy's ElasticKV pool and the Redis adapter's per-peer cap. +const DefaultElasticKVRedisConnections = 64 diff --git a/kv/coordinator.go b/kv/coordinator.go index 093d77686..77e96022a 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -36,15 +36,21 @@ const dispatchLeaderRetryBudget = 5 * time.Second const dispatchLeaderRetryInterval = 25 * time.Millisecond // hlcPhysicalWindowMs is the duration in milliseconds that the Raft-agreed -// physical ceiling extends ahead of the current wall clock. Modelled after -// TiDB's TSO 3-second window: the leader commits ceiling = now + window, and -// renews before the window expires. A new leader inherits the committed ceiling -// so it never issues timestamps that collide with the previous leader's window. -const hlcPhysicalWindowMs int64 = 3_000 +// physical ceiling extends ahead of the current wall clock. The leader commits +// ceiling = now + window, and renews before the window expires. A new leader +// inherits the committed ceiling so it never issues timestamps that collide +// with the previous leader's window. +const hlcPhysicalWindowMs int64 = 20_000 // hlcRenewalInterval controls how often the leader proposes a new ceiling. -// Must be less than hlcPhysicalWindowMs to guarantee the window never expires. -const hlcRenewalInterval = 1 * time.Second +// Keep the renewal interval and proposal timeout below the physical window; +// this provides timing margin but cannot prevent expiry after repeated failures. +const hlcRenewalInterval = 2 * time.Second + +// hlcRenewalProposalTimeout bounds one renewal proposal independently from the +// cadence. Under heavy Redis workloads, a safe quorum-acked renewal may need +// more than one tick; timing it out at the cadence can let the ceiling expire. +const hlcRenewalProposalTimeout = 5 * time.Second // CoordinatorOption is a functional option for Coordinate constructors. type CoordinatorOption func(*Coordinate) @@ -855,17 +861,20 @@ func (c *Coordinate) extendLeaseAfterRenewal(dispatchStart monoclock.Instant, ex // RunHLCLeaseRenewal runs a background loop that periodically proposes a new // physical ceiling to the Raft cluster while this node is the leader. // -// The ceiling is set to now + hlcPhysicalWindowMs (3 s) and is renewed every -// hlcRenewalInterval (1 s), mirroring TiDB's TSO window strategy. Because the -// window is always at least 2 s ahead of any real timestamp, a new leader will -// never issue timestamps that overlap with the previous leader's window. +// The ceiling is set to now + hlcPhysicalWindowMs and is renewed every +// hlcRenewalInterval. While renewals keep succeeding, the committed ceiling and +// NextFenced fail-closed check prevent timestamp issuance after the safe window +// has expired. // // RunHLCLeaseRenewal blocks until ctx is cancelled; call it in a goroutine. func (c *Coordinate) RunHLCLeaseRenewal(ctx context.Context) { - // Use a Timer rather than a Ticker so the next renewal is scheduled - // relative to the completion of the previous one. This prevents a burst - // of back-to-back proposals if ProposeHLCLease stalls (e.g. waiting for - // Raft quorum during a slow leader election). + if ctx == nil { + ctx = context.Background() + } + // Schedule relative to proposal launch, not completion. A renewal proposal + // may legitimately take longer than hlcRenewalInterval under load; letting + // the next tick launch a fresher ceiling keeps logical counter headroom + // available while the older proposal is still bounded by its own timeout. timer := time.NewTimer(hlcRenewalInterval) defer timer.Stop() for { @@ -876,13 +885,7 @@ func (c *Coordinate) RunHLCLeaseRenewal(ctx context.Context) { continue } if c.IsLeaderAcceptingWrites() { - ceilingMs := time.Now().UnixMilli() + hlcPhysicalWindowMs - if err := c.ProposeHLCLease(ctx, ceilingMs); err != nil { - c.log.WarnContext(ctx, "hlc lease renewal failed", - slog.Int64("ceiling_ms", ceilingMs), - slog.Any("err", err), - ) - } + c.renewHLCLeaseAsync(ctx) } timer.Reset(hlcRenewalInterval) case <-ctx.Done(): @@ -891,6 +894,20 @@ func (c *Coordinate) RunHLCLeaseRenewal(ctx context.Context) { } } +func (c *Coordinate) renewHLCLeaseAsync(ctx context.Context) { + ceilingMs := time.Now().UnixMilli() + hlcPhysicalWindowMs + go func() { + pctx, cancel := context.WithTimeout(ctx, hlcRenewalProposalTimeout) + defer cancel() + if err := c.ProposeHLCLease(pctx, ceilingMs); err != nil { + c.log.WarnContext(ctx, "hlc lease renewal failed", + slog.Int64("ceiling_ms", ceilingMs), + slog.Any("err", err), + ) + } + }() +} + func (c *Coordinate) IsLeaderForKey(_ []byte) bool { return c.IsLeader() } diff --git a/kv/lease_read_test.go b/kv/lease_read_test.go index 19bb598f2..3c2714ede 100644 --- a/kv/lease_read_test.go +++ b/kv/lease_read_test.go @@ -24,7 +24,8 @@ type fakeLeaseEngine struct { linearizableCalls atomic.Int32 proposeErr error // when set, Propose returns it (warm-up failure tests) proposeCalls atomic.Int32 - proposeHook func() // invoked inside Propose before returning (race injection) + proposeHook func() // invoked inside Propose before returning (race injection) + proposeCtxHook func(context.Context) proposeApply func([]byte) // invoked after a successful propose (FSM apply simulation) state atomic.Value // stores raftengine.State; default Leader lastQuorumAckMonoNs atomic.Int64 // 0 = no ack yet. Updated by setQuorumAck(). @@ -64,8 +65,11 @@ func (e *fakeLeaseEngine) Status() raftengine.Status { func (e *fakeLeaseEngine) Configuration(context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } -func (e *fakeLeaseEngine) Propose(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { +func (e *fakeLeaseEngine) Propose(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { e.proposeCalls.Add(1) + if e.proposeCtxHook != nil { + e.proposeCtxHook(ctx) + } if e.proposeHook != nil { e.proposeHook() } diff --git a/kv/lease_warmup_test.go b/kv/lease_warmup_test.go index e3ca74f79..fdb79806b 100644 --- a/kv/lease_warmup_test.go +++ b/kv/lease_warmup_test.go @@ -170,6 +170,30 @@ func TestCoordinate_RunHLCLeaseRenewal_BlockerSuppressesProposals(t *testing.T) "HLC renewal should resume after startup rotation blocker clears") } +func TestCoordinate_RenewHLCLeaseAsync_OverlapsSlowProposal(t *testing.T) { + eng := &fakeLeaseEngine{applied: 11, leaseDur: time.Hour} + entered := make(chan struct{}, 2) + release := make(chan struct{}) + eng.proposeHook = func() { + entered <- struct{}{} + <-release + } + c := NewCoordinatorWithEngine(nil, eng) + + c.renewHLCLeaseAsync(context.Background()) + <-entered + c.renewHLCLeaseAsync(context.Background()) + <-entered + + require.Equal(t, int32(2), eng.proposeCalls.Load(), + "a slow renewal proposal must not monopolize newer ceiling proposals") + close(release) + require.Eventually(t, func() bool { + return c.lease.valid(monoclock.Now()) + }, time.Second, 10*time.Millisecond, + "overlapped renewal goroutines should finish and warm the lease") +} + // TestShardedCoordinator_RenewHLCLease_WarmsGroupLease proves the // sharded renewal path warms the target group's lease on a successful // propose, so LeaseReadForKey on a key owned by that group serves from the @@ -392,14 +416,48 @@ func TestShardedCoordinator_RenewHLCLeases_SlowGroupDoesNotBlockPeers(t *testing requireRenewalDone(t, done) } -func TestShardedCoordinator_RenewHLCLeases_SkipsInFlightGroup(t *testing.T) { +func TestShardedCoordinator_RenewHLCLeases_UsesProposalTimeoutBeyondCadence(t *testing.T) { + t.Parallel() eng1 := newShardedLeaseEngine(100) eng2 := newShardedLeaseEngine(200) - entered := make(chan struct{}) + deadlineRemaining := make(chan time.Duration, 1) + eng1.proposeCtxHook = func(ctx context.Context) { + deadline, ok := ctx.Deadline() + if !ok { + deadlineRemaining <- 0 + return + } + deadlineRemaining <- time.Until(deadline) + } + coord := mustShardedLeaseCoord(t, eng1, eng2) + + done := coord.renewHLCLeases(context.Background()) + requireRenewalDone(t, done) + + select { + case remaining := <-deadlineRemaining: + require.Greater(t, remaining, hlcRenewalInterval, + "renewal proposals need a timeout longer than the cadence under load") + require.LessOrEqual(t, remaining, hlcRenewalProposalTimeout) + case <-time.After(time.Second): + t.Fatal("timed out waiting for captured renewal deadline") + } +} + +func TestHLCLeaseRenewalTimingHasPhysicalWindowMargin(t *testing.T) { + t.Parallel() + window := time.Duration(hlcPhysicalWindowMs) * time.Millisecond + require.Less(t, hlcRenewalInterval+hlcRenewalProposalTimeout, window) + require.Less(t, uint64(hlcPhysicalWindowMs), defaultTxnLockTTLms) +} + +func TestShardedCoordinator_RenewHLCLeases_OverlapsInFlightGroup(t *testing.T) { + eng1 := newShardedLeaseEngine(100) + eng2 := newShardedLeaseEngine(200) + entered := make(chan struct{}, 2) release := make(chan struct{}) - var enteredOnce sync.Once eng1.proposeHook = func() { - enteredOnce.Do(func() { close(entered) }) + entered <- struct{}{} <-release } coord := mustShardedLeaseCoord(t, eng1, eng2) @@ -407,25 +465,30 @@ func TestShardedCoordinator_RenewHLCLeases_SkipsInFlightGroup(t *testing.T) { first := coord.renewHLCLeases(context.Background()) <-entered require.Eventually(t, func() bool { - return eng2.proposeCalls.Load() == 1 && !hlcRenewalInFlight(coord, 2) + return eng2.proposeCalls.Load() == 1 }, time.Second, 10*time.Millisecond, "precondition: the first round must fully finish for the non-blocked group") second := coord.renewHLCLeases(context.Background()) - requireRenewalDone(t, second) + <-entered - require.Equal(t, int32(1), eng1.proposeCalls.Load(), - "an in-flight group must not receive a second concurrent renewal proposal") + require.Eventually(t, func() bool { + return eng1.proposeCalls.Load() == 2 && eng2.proposeCalls.Load() == 2 + }, time.Second, 10*time.Millisecond, + "the next renewal round should launch for both the slow group and its peer") + require.Equal(t, int32(2), eng1.proposeCalls.Load(), + "a slow in-flight group still needs a fresher renewal proposal on the next tick") require.Equal(t, int32(2), eng2.proposeCalls.Load(), "other led groups must still renew while one group is in flight") close(release) requireRenewalDone(t, first) + requireRenewalDone(t, second) third := coord.renewHLCLeases(context.Background()) requireRenewalDone(t, third) - require.Equal(t, int32(2), eng1.proposeCalls.Load(), - "the group must be eligible for renewal after the in-flight proposal finishes") + require.Equal(t, int32(3), eng1.proposeCalls.Load(), + "the group remains eligible for later renewal after overlapping proposals finish") } func TestShardedCoordinator_ProposeHLCLease_UsesDedicatedTimestampGroup(t *testing.T) { @@ -447,13 +510,6 @@ func TestShardedCoordinator_ProposeHLCLease_UsesDedicatedTimestampGroup(t *testi "a synchronous timestamp renewal must warm the timestamp group's lease") } -func hlcRenewalInFlight(coord *ShardedCoordinator, gid uint64) bool { - coord.hlcRenewalMu.Lock() - defer coord.hlcRenewalMu.Unlock() - _, ok := coord.hlcRenewalInFlight[gid] - return ok -} - func requireRenewalDone(t *testing.T, done <-chan struct{}) { t.Helper() select { diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index fdf3e1ec3..f23855a65 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -409,10 +409,6 @@ type ShardedCoordinator struct { // lease proposals while another startup-only Raft mutation must run first. hlcRenewalBlocked func() bool hlcRecoveryMu sync.Mutex - // hlcRenewalInFlight prevents a slow or quorum-stalled group from stacking - // another background HLC lease proposal for the same group on the next tick. - hlcRenewalMu sync.Mutex - hlcRenewalInFlight map[uint64]struct{} // registrationGate is the Stage 7a §4.1 first-write barrier: when // set, self-originated mutating writes that would land as §4.1 // storage envelopes block until this node's writer registration @@ -2370,8 +2366,10 @@ func (c *ShardedCoordinator) timestampLeaseRenewalGroupIDs() []uint64 { } // renewHLCLeases starts one renewal proposal for every shard group this node -// currently leads. It does not wait for those proposals before returning; the -// returned channel closes when the launched proposals finish and exists for +// currently leads. It does not suppress a group that still has an older renewal +// proposal in flight: HLC lease entries apply monotonically, so overlapping +// proposals are safe, and each proposal's context bounds the maximum overlap. +// The returned channel closes when the launched proposals finish and exists for // tests/diagnostics only. func (c *ShardedCoordinator) renewHLCLeases(ctx context.Context) <-chan struct{} { if ctx == nil { @@ -2386,14 +2384,10 @@ func (c *ShardedCoordinator) renewHLCLeases(ctx context.Context) <-chan struct{} if !shardGroupAcceptingWrites(group) { continue } - if !c.startHLCLeaseRenewal(gid) { - continue - } wg.Add(1) go func(gid uint64, group *ShardGroup) { defer wg.Done() - defer c.finishHLCLeaseRenewal(gid) - pctx, cancel := context.WithTimeout(ctx, hlcRenewalInterval) + pctx, cancel := context.WithTimeout(ctx, hlcRenewalProposalTimeout) defer cancel() c.renewHLCLease(pctx, gid, group) }(gid, group) @@ -2405,25 +2399,6 @@ func (c *ShardedCoordinator) renewHLCLeases(ctx context.Context) <-chan struct{} return done } -func (c *ShardedCoordinator) startHLCLeaseRenewal(gid uint64) bool { - c.hlcRenewalMu.Lock() - defer c.hlcRenewalMu.Unlock() - if c.hlcRenewalInFlight == nil { - c.hlcRenewalInFlight = make(map[uint64]struct{}) - } - if _, ok := c.hlcRenewalInFlight[gid]; ok { - return false - } - c.hlcRenewalInFlight[gid] = struct{}{} - return true -} - -func (c *ShardedCoordinator) finishHLCLeaseRenewal(gid uint64) { - c.hlcRenewalMu.Lock() - defer c.hlcRenewalMu.Unlock() - delete(c.hlcRenewalInFlight, gid) -} - // renewHLCLease proposes a fresh physical ceiling on one shard // group and, on a successful (quorum-acked) propose, warms that group's // read lease. diff --git a/proxy/backend.go b/proxy/backend.go index 704cfcc45..1ee94e4da 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -6,12 +6,13 @@ import ( "fmt" "time" + "github.com/bootjp/elastickv/internal/redislimits" "github.com/redis/go-redis/v9" ) const ( defaultPoolSize = 128 - defaultElasticKVPoolSize = 64 + defaultElasticKVPoolSize = redislimits.DefaultElasticKVRedisConnections defaultDialTimeout = 5 * time.Second defaultReadTimeout = 3 * time.Second defaultWriteTimeout = 3 * time.Second diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index 34c1c3259..758017918 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -32,8 +32,9 @@ const ( // maxBlockingReplayGoroutines isolates mutating blocking command replays // from normal secondary writes. Blocking replays may wait for the secondary // to observe a producer write, and they hit the secondary's heavy-command - // limiter, so keep the default well below the normal write limit. - maxBlockingReplayGoroutines = 20 + // limiter, so the command caps the derived value by the remaining backend + // connection budget. + maxBlockingReplayGoroutines = 64 // Async queues absorb short bursts without allowing an unavailable or slow // secondary to build an unbounded replay backlog. minAsyncQueueCapacity = 64 diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 22936507f..43bdb0507 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/bootjp/elastickv/internal/redislimits" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/redis/go-redis/v9" @@ -1363,7 +1364,7 @@ func TestDefaultBackendOptions(t *testing.T) { func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() - assert.Equal(t, 64, opts.PoolSize) + assert.Equal(t, redislimits.DefaultElasticKVRedisConnections, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) assert.Equal(t, 3*time.Second, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout)