From 10619550977d8e0cb8d429e7018e4588d73b4b37 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 11:42:51 +0900 Subject: [PATCH 01/18] redis-proxy: raise ElasticKV connection budget --- README.md | 6 +++--- adapter/redis_peer_limiter.go | 7 ++++--- adapter/redis_peer_limiter_test.go | 6 +++++- cmd/redis-proxy/main_test.go | 25 ++++++++++++++++++++++++ deploy/redis-proxy/docker-compose.ha.yml | 4 ++-- docs/redis-proxy-deployment.md | 23 ++++++++++++---------- proxy/backend.go | 11 ++++++----- proxy/proxy_test.go | 2 +- 8 files changed, 59 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index fd3fab601..304374fb3 100644 --- a/README.md +++ b/README.md @@ -202,9 +202,9 @@ docker run --rm \ -listen :6479 \ -primary redis.internal:6379 \ -secondary elastickv.internal:6380 \ - -elastickv-pool-size 4 \ - -secondary-write-concurrency 2 \ - -secondary-script-concurrency 1 \ + -elastickv-pool-size 128 \ + -secondary-write-concurrency 64 \ + -secondary-script-concurrency 32 \ -mode dual-write ``` diff --git a/adapter/redis_peer_limiter.go b/adapter/redis_peer_limiter.go index 88a3077fa..e76a5f15a 100644 --- a/adapter/redis_peer_limiter.go +++ b/adapter/redis_peer_limiter.go @@ -10,9 +10,10 @@ import ( const ( redisPerPeerLimitEnv = "ELASTICKV_REDIS_PER_PEER_CONNECTIONS" - defaultRedisProxyPoolPeerCap = 64 - defaultRedisDedicatedPeerHeadroom = 64 - defaultRedisPerPeerConnectionCap = defaultRedisProxyPoolPeerCap + defaultRedisDedicatedPeerHeadroom + defaultRedisProxyPoolPeerCap = 128 + defaultRedisProxyReplicasPerPeer = 2 + defaultRedisDedicatedPeerHeadroom = 128 + defaultRedisPerPeerConnectionCap = defaultRedisProxyPoolPeerCap*defaultRedisProxyReplicasPerPeer + defaultRedisDedicatedPeerHeadroom redisPeerLimitError = "ERR max connections per client exceeded" unknownRedisPeer = "unknown" ) diff --git a/adapter/redis_peer_limiter_test.go b/adapter/redis_peer_limiter_test.go index d40f23b58..f602e520a 100644 --- a/adapter/redis_peer_limiter_test.go +++ b/adapter/redis_peer_limiter_test.go @@ -15,7 +15,11 @@ func TestRedisPeerLimiterDefaultMatchesProxyPool(t *testing.T) { t.Setenv(redisPerPeerLimitEnv, "") limiter := newDefaultRedisPeerLimiter() require.NotNil(t, limiter) - require.Equal(t, proxy.DefaultElasticKVBackendOptions().PoolSize+defaultRedisDedicatedPeerHeadroom, limiter.limit) + require.Equal(t, proxy.DefaultElasticKVBackendOptions().PoolSize, defaultRedisProxyPoolPeerCap) + require.Equal(t, + proxy.DefaultElasticKVBackendOptions().PoolSize*defaultRedisProxyReplicasPerPeer+defaultRedisDedicatedPeerHeadroom, + limiter.limit, + ) } func TestRedisPeerLimiterRejectsAndReleases(t *testing.T) { diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 7188637f9..9a92951cc 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -154,3 +154,28 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { }) } } + +func TestDeriveSecondaryConcurrencyFromDefaultElasticKVPool(t *testing.T) { + poolSize := proxy.DefaultElasticKVBackendOptions().PoolSize + writeConcurrency, scriptConcurrency, blockingConcurrency := deriveSecondaryConcurrency( + proxy.ModeDualWrite, + proxy.DefaultBackendOptions().PoolSize, + poolSize, + 0, + 0, + 0, + ) + + require.Equal(t, 128, poolSize) + require.Equal(t, 64, writeConcurrency) + require.Equal(t, 32, scriptConcurrency) + require.Equal(t, 20, blockingConcurrency) + require.NoError(t, validateSecondaryConcurrency( + proxy.ModeDualWrite, + proxy.DefaultBackendOptions().PoolSize, + poolSize, + writeConcurrency, + scriptConcurrency, + blockingConcurrency, + )) +} diff --git a/deploy/redis-proxy/docker-compose.ha.yml b/deploy/redis-proxy/docker-compose.ha.yml index 1376c0a84..2c89f6ee5 100644 --- a/deploy/redis-proxy/docker-compose.ha.yml +++ b/deploy/redis-proxy/docker-compose.ha.yml @@ -26,7 +26,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-64} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-128} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: @@ -46,7 +46,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-64} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-128} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index 1f0468936..7fb62e1e2 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -35,11 +35,13 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-secondary-db` | `0` | Secondary Redis DB number | | `-secondary-password` | (empty) | Secondary Redis password | | `-primary-pool-size` | `128` | Primary Redis backend connection pool size | -| `-elastickv-pool-size` | `64` | ElasticKV backend connection pool size | +| `-elastickv-pool-size` | `128` | ElasticKV backend connection pool size | | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` | | `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` | +| `-secondary-blocking-replay-concurrency` | `0` | Mutating blocking-command replay sublimit. `0` uses remaining secondary backend pool capacity, capped at `20` | | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-secondary-script-queue-size` | `0` | Bounded queue for secondary Lua-script writes. `0` derives `64 * concurrency`, clamped to `64..8192` | +| `-secondary-blocking-replay-queue-size` | `0` | Bounded queue for mutating blocking-command replays. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-mode` | `dual-write` | Proxy mode (see below) | | `-secondary-timeout` | `5s` | End-to-end secondary write deadline, including queue wait | | `-shadow-timeout` | `3s` | Shadow read timeout | @@ -94,9 +96,9 @@ docker run --rm \ -primary redis.internal:6379 \ -primary-password "${REDIS_PASSWORD}" \ -secondary elastickv.internal:6380 \ - -elastickv-pool-size 64 \ - -secondary-write-concurrency 32 \ - -secondary-script-concurrency 16 \ + -elastickv-pool-size 128 \ + -secondary-write-concurrency 64 \ + -secondary-script-concurrency 32 \ -mode dual-write-shadow \ -secondary-timeout 5s \ -shadow-timeout 3s \ @@ -118,9 +120,9 @@ services: - -listen=:6479 - -primary=redis:6379 - -secondary=elastickv:6380 - - -elastickv-pool-size=64 - - -secondary-write-concurrency=32 - - -secondary-script-concurrency=16 + - -elastickv-pool-size=128 + - -secondary-write-concurrency=64 + - -secondary-script-concurrency=32 - -mode=dual-write-shadow - -metrics=:9191 depends_on: @@ -213,7 +215,7 @@ Override backend wiring via env vars before `docker compose up`: ```bash REDIS_PROXY_PRIMARY=redis.prod.internal:6379 \ REDIS_PROXY_SECONDARY=elastickv-1.prod.internal:6380,elastickv-2.prod.internal:6380,elastickv-3.prod.internal:6380 \ -REDIS_PROXY_ELASTICKV_POOL_SIZE=64 \ +REDIS_PROXY_ELASTICKV_POOL_SIZE=128 \ REDIS_PROXY_MODE=dual-write-shadow \ docker compose -f docker-compose.ha.yml up -d ``` @@ -414,7 +416,8 @@ groups: | Parameter | Value | Description | |-----------|-------|-------------| | Redis connection pool size | 128 | Default go-redis pool size for Redis | -| ElasticKV connection pool size | 64 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections | +| ElasticKV connection pool size | 128 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections | +| ElasticKV Redis per-peer connection cap | 384 | Default server-side cap: two proxy replicas at pool `128`, plus dedicated PubSub/shadow PubSub headroom | | Dial timeout | 5s | Backend connection timeout | | Read timeout | 3s | Backend read timeout | | Write timeout | 3s | Backend write timeout | @@ -439,7 +442,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. ### Secondary writes are falling behind - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. -- Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` above `-elastickv-pool-size`; PubSub and shadow PubSub use dedicated connections outside the command pool. Keep `-secondary-write-concurrency` at or below the pool size. +- Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` at least `proxy replicas sharing one client IP * -elastickv-pool-size + 128`; PubSub and shadow PubSub use dedicated connections outside the command pool. With the HA compose defaults, use at least `384`. Keep `-secondary-write-concurrency` at or below the pool size. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. ### High divergence count diff --git a/proxy/backend.go b/proxy/backend.go index 599e995dc..3774a3cdb 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -11,7 +11,7 @@ import ( const ( defaultPoolSize = 128 - defaultElasticKVPoolSize = 64 + defaultElasticKVPoolSize = 128 defaultDialTimeout = 5 * time.Second defaultReadTimeout = 3 * time.Second defaultWriteTimeout = 3 * time.Second @@ -73,10 +73,11 @@ func DefaultBackendOptions() BackendOptions { // DefaultElasticKVBackendOptions returns defaults for proxy backends that // connect to ElasticKV's Redis adapter. Production dual-write deployments -// should run the cluster with ELASTICKV_REDIS_PER_PEER_CONNECTIONS above this -// pool size because PubSub and shadow PubSub use dedicated connections outside -// the go-redis command pool. Lower the proxy pool instead for clusters that -// keep the server-side per-peer cap below the default. +// should run the cluster with ELASTICKV_REDIS_PER_PEER_CONNECTIONS sized for +// every proxy replica that may share one client IP, plus dedicated PubSub and +// shadow PubSub connections outside this command pool. Lower the proxy pool +// instead for clusters that keep the server-side per-peer cap below the +// default. func DefaultElasticKVBackendOptions() BackendOptions { opts := DefaultBackendOptions() opts.PoolSize = defaultElasticKVPoolSize diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 94cac2cf4..a67ec86ec 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -1334,7 +1334,7 @@ func TestDefaultBackendOptions(t *testing.T) { func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() - assert.Equal(t, 64, opts.PoolSize) + assert.Equal(t, 128, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) assert.Equal(t, 3*time.Second, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout) From b4a9a63f25c06d80ef1f36e20216266fbe799eff Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 12:16:10 +0900 Subject: [PATCH 02/18] redis-proxy: tune ElasticKV backpressure budget --- README.md | 6 ++--- adapter/redis.go | 4 +++ adapter/redis_lua.go | 4 +-- adapter/redis_lua_context.go | 2 +- adapter/redis_peer_limiter.go | 2 +- cmd/redis-proxy/main.go | 18 +++++++++----- cmd/redis-proxy/main_test.go | 24 +++++++++--------- deploy/redis-proxy/docker-compose.ha.yml | 4 +-- docs/redis-proxy-deployment.md | 31 ++++++++++++------------ proxy/backend.go | 2 +- proxy/config.go | 2 +- proxy/dualwrite.go | 2 +- proxy/proxy_test.go | 3 ++- 13 files changed, 58 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 304374fb3..26a9f233a 100644 --- a/README.md +++ b/README.md @@ -202,9 +202,9 @@ docker run --rm \ -listen :6479 \ -primary redis.internal:6379 \ -secondary elastickv.internal:6380 \ - -elastickv-pool-size 128 \ - -secondary-write-concurrency 64 \ - -secondary-script-concurrency 32 \ + -elastickv-pool-size 192 \ + -secondary-write-concurrency 96 \ + -secondary-script-concurrency 2 \ -mode dual-write ``` diff --git a/adapter/redis.go b/adapter/redis.go index 01a7cdf96..87dd78dcb 100644 --- a/adapter/redis.go +++ b/adapter/redis.go @@ -116,6 +116,10 @@ const ( const ( redisDispatchTimeout = 10 * time.Second + // redisLuaDispatchTimeout gives EVAL/EVALSHA enough room for migration + // scripts that expand into thousands of Redis calls while keeping regular + // commands on the tighter dispatch deadline. + redisLuaDispatchTimeout = 30 * time.Second // defaultRedisBlockWaitFallback is the safety-net poll interval for // blocking-command wait loops when no in-process write signal arrives. // Signals cover normal XADD / ZADD / ZINCRBY wakeups immediately; this diff --git a/adapter/redis_lua.go b/adapter/redis_lua.go index 08a46674f..d6792e937 100644 --- a/adapter/redis_lua.go +++ b/adapter/redis_lua.go @@ -115,7 +115,7 @@ func (r *RedisServer) runLuaScript(conn redcon.Conn, script string, evalArgs [][ return } - ctx, cancel := context.WithTimeout(r.handlerContext(), redisDispatchTimeout) + ctx, cancel := context.WithTimeout(r.handlerContext(), redisLuaDispatchTimeout) defer cancel() start := time.Now() @@ -833,7 +833,7 @@ func (r *RedisServer) execLuaCompat(conn redcon.Conn, command string, args [][]b stringArgs[i] = string(arg) } - ctx, cancel := context.WithTimeout(context.Background(), redisDispatchTimeout) + ctx, cancel := context.WithTimeout(context.Background(), redisLuaDispatchTimeout) defer cancel() var reply luaReply diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 94cd49627..dd35b9dfa 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -3534,7 +3534,7 @@ func (c *luaScriptContext) commit() error { } sort.Strings(keys) - ctx, cancel := context.WithTimeout(c.scriptCtx(), redisDispatchTimeout) + ctx, cancel := context.WithTimeout(c.scriptCtx(), redisLuaDispatchTimeout) defer cancel() // Pre-allocate a commitTS so Delta key bytes can embed it before dispatch. diff --git a/adapter/redis_peer_limiter.go b/adapter/redis_peer_limiter.go index e76a5f15a..84be9f9b4 100644 --- a/adapter/redis_peer_limiter.go +++ b/adapter/redis_peer_limiter.go @@ -10,7 +10,7 @@ import ( const ( redisPerPeerLimitEnv = "ELASTICKV_REDIS_PER_PEER_CONNECTIONS" - defaultRedisProxyPoolPeerCap = 128 + defaultRedisProxyPoolPeerCap = 192 defaultRedisProxyReplicasPerPeer = 2 defaultRedisDedicatedPeerHeadroom = 128 defaultRedisPerPeerConnectionCap = defaultRedisProxyPoolPeerCap*defaultRedisProxyReplicasPerPeer + defaultRedisDedicatedPeerHeadroom diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index f7f2358a7..2ae7aedeb 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -19,9 +19,11 @@ import ( ) const ( - sentryFlushTimeout = 2 * time.Second - metricsShutdownTimeout = 5 * time.Second - secondaryConcurrencyDivisor = 2 + sentryFlushTimeout = 2 * time.Second + metricsShutdownTimeout = 5 * time.Second + secondaryWriteConcurrencyDivisor = 2 + secondaryScriptConcurrencyDivisor = 32 + secondaryScriptConcurrencyCap = 2 ) func main() { @@ -53,7 +55,7 @@ func run() error { flag.IntVar(&primaryPoolSize, "primary-pool-size", primaryPoolSize, "Primary Redis backend connection pool size") flag.IntVar(&elasticKVPoolSize, "elastickv-pool-size", elasticKVPoolSize, "ElasticKV backend connection pool size") flag.IntVar(&secondaryWriteConcurrency, "secondary-write-concurrency", secondaryWriteConcurrency, "Maximum concurrent asynchronous secondary writes including scripts (0 = half of secondary backend pool size)") - flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = half of secondary write concurrency)") + flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = secondary write concurrency / 32, capped at 2)") flag.IntVar(&secondaryBlockingReplayConcurrency, "secondary-blocking-replay-concurrency", secondaryBlockingReplayConcurrency, "Maximum concurrent asynchronous secondary mutating blocking-command replays (0 = capped remaining secondary backend pool capacity after writes)") flag.IntVar(&secondaryWriteQueueSize, "secondary-write-queue-size", secondaryWriteQueueSize, "Maximum queued asynchronous secondary writes (0 = derived from write concurrency)") flag.IntVar(&secondaryScriptQueueSize, "secondary-script-queue-size", secondaryScriptQueueSize, "Maximum queued asynchronous secondary Lua-script writes (0 = derived from script concurrency)") @@ -283,11 +285,15 @@ func secondaryBackendPoolSize(mode proxy.ProxyMode, primaryPoolSize, elasticKVPo } func defaultSecondaryWriteConcurrency(poolSize int) int { - return atLeastOne(poolSize / secondaryConcurrencyDivisor) + return atLeastOne(poolSize / secondaryWriteConcurrencyDivisor) } func defaultSecondaryScriptConcurrency(writeConcurrency int) int { - return atLeastOne(writeConcurrency / secondaryConcurrencyDivisor) + concurrency := atLeastOne(writeConcurrency / secondaryScriptConcurrencyDivisor) + if concurrency > secondaryScriptConcurrencyCap { + return secondaryScriptConcurrencyCap + } + return concurrency } func defaultSecondaryBlockingReplayConcurrency(poolSize, writeConcurrency int) int { diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 9a92951cc..7e566f3f7 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -74,8 +74,8 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { primaryPoolSize: 128, elasticKVPoolSize: 64, wantWriteConcurrency: 32, - wantScriptConcurrency: 16, - wantBlockingConcurrency: 20, + wantScriptConcurrency: 1, + wantBlockingConcurrency: 32, }, { name: "shadow mode derives from ElasticKV pool", @@ -83,7 +83,7 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { primaryPoolSize: 128, elasticKVPoolSize: 8, wantWriteConcurrency: 4, - wantScriptConcurrency: 2, + wantScriptConcurrency: 1, wantBlockingConcurrency: 4, }, { @@ -92,8 +92,8 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { primaryPoolSize: 128, elasticKVPoolSize: 4, wantWriteConcurrency: 64, - wantScriptConcurrency: 32, - wantBlockingConcurrency: 20, + wantScriptConcurrency: 2, + wantBlockingConcurrency: 32, }, { name: "large remaining pool caps blocking replay", @@ -102,8 +102,8 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { elasticKVPoolSize: 144, writeConcurrency: 80, wantWriteConcurrency: 80, - wantScriptConcurrency: 40, - wantBlockingConcurrency: 20, + wantScriptConcurrency: 2, + wantBlockingConcurrency: 32, }, { name: "explicit write keeps derived script", @@ -112,7 +112,7 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { elasticKVPoolSize: 8, writeConcurrency: 5, wantWriteConcurrency: 5, - wantScriptConcurrency: 2, + wantScriptConcurrency: 1, wantBlockingConcurrency: 3, }, { @@ -166,10 +166,10 @@ func TestDeriveSecondaryConcurrencyFromDefaultElasticKVPool(t *testing.T) { 0, ) - require.Equal(t, 128, poolSize) - require.Equal(t, 64, writeConcurrency) - require.Equal(t, 32, scriptConcurrency) - require.Equal(t, 20, blockingConcurrency) + require.Equal(t, 192, poolSize) + require.Equal(t, 96, writeConcurrency) + require.Equal(t, 2, scriptConcurrency) + require.Equal(t, 32, blockingConcurrency) require.NoError(t, validateSecondaryConcurrency( proxy.ModeDualWrite, proxy.DefaultBackendOptions().PoolSize, diff --git a/deploy/redis-proxy/docker-compose.ha.yml b/deploy/redis-proxy/docker-compose.ha.yml index 2c89f6ee5..3ce357907 100644 --- a/deploy/redis-proxy/docker-compose.ha.yml +++ b/deploy/redis-proxy/docker-compose.ha.yml @@ -26,7 +26,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-128} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-192} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: @@ -46,7 +46,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-128} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-192} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index 7fb62e1e2..65c9026f6 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -35,15 +35,15 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-secondary-db` | `0` | Secondary Redis DB number | | `-secondary-password` | (empty) | Secondary Redis password | | `-primary-pool-size` | `128` | Primary Redis backend connection pool size | -| `-elastickv-pool-size` | `128` | ElasticKV backend connection pool size | +| `-elastickv-pool-size` | `192` | ElasticKV backend connection pool size | | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` | -| `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` | -| `-secondary-blocking-replay-concurrency` | `0` | Mutating blocking-command replay sublimit. `0` uses remaining secondary backend pool capacity, capped at `20` | +| `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives one thirty-second of the shared write limit, capped at `2`, minimum `1` | +| `-secondary-blocking-replay-concurrency` | `0` | Mutating blocking-command replay sublimit. `0` uses remaining secondary backend pool capacity, capped at `32` | | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-secondary-script-queue-size` | `0` | Bounded queue for secondary Lua-script writes. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-secondary-blocking-replay-queue-size` | `0` | Bounded queue for mutating blocking-command replays. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-mode` | `dual-write` | Proxy mode (see below) | -| `-secondary-timeout` | `5s` | End-to-end secondary write deadline, including queue wait | +| `-secondary-timeout` | `30s` | End-to-end secondary write deadline, including queue wait | | `-shadow-timeout` | `3s` | Shadow read timeout | | `-sentry-dsn` | (empty) | Sentry DSN (empty = disabled) | | `-sentry-env` | (empty) | Sentry environment name | @@ -96,11 +96,11 @@ docker run --rm \ -primary redis.internal:6379 \ -primary-password "${REDIS_PASSWORD}" \ -secondary elastickv.internal:6380 \ - -elastickv-pool-size 128 \ - -secondary-write-concurrency 64 \ - -secondary-script-concurrency 32 \ + -elastickv-pool-size 192 \ + -secondary-write-concurrency 96 \ + -secondary-script-concurrency 2 \ -mode dual-write-shadow \ - -secondary-timeout 5s \ + -secondary-timeout 30s \ -shadow-timeout 3s \ -sentry-dsn "${SENTRY_DSN}" \ -sentry-env production \ @@ -120,9 +120,9 @@ services: - -listen=:6479 - -primary=redis:6379 - -secondary=elastickv:6380 - - -elastickv-pool-size=128 - - -secondary-write-concurrency=64 - - -secondary-script-concurrency=32 + - -elastickv-pool-size=192 + - -secondary-write-concurrency=96 + - -secondary-script-concurrency=2 - -mode=dual-write-shadow - -metrics=:9191 depends_on: @@ -215,7 +215,7 @@ Override backend wiring via env vars before `docker compose up`: ```bash REDIS_PROXY_PRIMARY=redis.prod.internal:6379 \ REDIS_PROXY_SECONDARY=elastickv-1.prod.internal:6380,elastickv-2.prod.internal:6380,elastickv-3.prod.internal:6380 \ -REDIS_PROXY_ELASTICKV_POOL_SIZE=128 \ +REDIS_PROXY_ELASTICKV_POOL_SIZE=192 \ REDIS_PROXY_MODE=dual-write-shadow \ docker compose -f docker-compose.ha.yml up -d ``` @@ -416,8 +416,8 @@ groups: | Parameter | Value | Description | |-----------|-------|-------------| | Redis connection pool size | 128 | Default go-redis pool size for Redis | -| ElasticKV connection pool size | 128 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections | -| ElasticKV Redis per-peer connection cap | 384 | Default server-side cap: two proxy replicas at pool `128`, plus dedicated PubSub/shadow PubSub headroom | +| ElasticKV connection pool size | 192 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections | +| ElasticKV Redis per-peer connection cap | 512 | Default server-side cap: two proxy replicas at pool `192`, plus dedicated PubSub/shadow PubSub headroom | | Dial timeout | 5s | Backend connection timeout | | Read timeout | 3s | Backend read timeout | | Write timeout | 3s | Backend write timeout | @@ -442,7 +442,8 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. ### Secondary writes are falling behind - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. -- Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` at least `proxy replicas sharing one client IP * -elastickv-pool-size + 128`; PubSub and shadow PubSub use dedicated connections outside the command pool. With the HA compose defaults, use at least `384`. Keep `-secondary-write-concurrency` at or below the pool size. +- Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` at least `proxy replicas sharing one client IP * -elastickv-pool-size + 128`; PubSub and shadow PubSub use dedicated connections outside the command pool. With the HA compose defaults, use at least `512`. Keep `-secondary-write-concurrency` at or below the pool size. +- If script drops rise while backend pool waits stay at zero, the bottleneck is server-side Lua replay or wide-column cleanup, not connection acquisition. Keep `-secondary-script-concurrency` low and profile ElasticKV before raising it. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. ### High divergence count diff --git a/proxy/backend.go b/proxy/backend.go index 3774a3cdb..bd9394a93 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -11,7 +11,7 @@ import ( const ( defaultPoolSize = 128 - defaultElasticKVPoolSize = 128 + defaultElasticKVPoolSize = 192 defaultDialTimeout = 5 * time.Second defaultReadTimeout = 3 * time.Second defaultWriteTimeout = 3 * time.Second diff --git a/proxy/config.go b/proxy/config.go index e99229ff9..dfe47b724 100644 --- a/proxy/config.go +++ b/proxy/config.go @@ -3,7 +3,7 @@ package proxy import "time" const ( - defaultSecondaryTimeout = 5 * time.Second + defaultSecondaryTimeout = 30 * time.Second defaultShadowTimeout = 3 * time.Second defaultPubSubCompareWindow = 2 * time.Second defaultPubSubSweepInterval = 500 * time.Millisecond diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index dc1e16bf7..d6741a8f7 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -33,7 +33,7 @@ const ( // from normal secondary writes. Blocking replays may wait for the secondary // to observe a producer write, and they hit the secondary's heavy-command // limiter, so keep the default well below the normal write limit. - maxBlockingReplayGoroutines = 20 + maxBlockingReplayGoroutines = 32 // Async queues absorb short bursts without allowing an unavailable or slow // secondary to build an unbounded replay backlog. minAsyncQueueCapacity = 64 diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index a67ec86ec..623e7a4a6 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -183,6 +183,7 @@ func TestDefaultConfig(t *testing.T) { assert.Equal(t, "localhost:6379", cfg.PrimaryAddr) assert.Equal(t, "localhost:6380", cfg.SecondaryAddr) assert.Equal(t, ModeDualWrite, cfg.Mode) + assert.Equal(t, 30*time.Second, cfg.SecondaryTimeout) } // ========== compare.go tests ========== @@ -1334,7 +1335,7 @@ func TestDefaultBackendOptions(t *testing.T) { func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() - assert.Equal(t, 128, opts.PoolSize) + assert.Equal(t, 192, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) assert.Equal(t, 3*time.Second, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout) From 7e4e5dc46d28452d1221a3ff8c92196e5814694b Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 12:22:09 +0900 Subject: [PATCH 03/18] redis-proxy: add script replay timeout --- cmd/redis-proxy/main.go | 1 + docs/redis-proxy-deployment.md | 4 +++- proxy/async_queue.go | 8 +++++-- proxy/config.go | 13 +++++++---- proxy/dualwrite.go | 12 ++++++++-- proxy/proxy_test.go | 42 ++++++++++++++++++++++++++++++++++ 6 files changed, 71 insertions(+), 9 deletions(-) diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index 2ae7aedeb..108d77639 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -62,6 +62,7 @@ func run() error { flag.IntVar(&secondaryBlockingReplayQueueSize, "secondary-blocking-replay-queue-size", secondaryBlockingReplayQueueSize, "Maximum queued asynchronous secondary mutating blocking-command replays (0 = derived from blocking replay concurrency)") flag.StringVar(&modeStr, "mode", "dual-write", "Proxy mode: redis-only, dual-write, dual-write-shadow, elastickv-primary, elastickv-only") flag.DurationVar(&cfg.SecondaryTimeout, "secondary-timeout", cfg.SecondaryTimeout, "Secondary write timeout") + flag.DurationVar(&cfg.SecondaryScriptTimeout, "secondary-script-timeout", cfg.SecondaryScriptTimeout, "Secondary Lua-script write timeout (0 = secondary-timeout)") flag.DurationVar(&cfg.ShadowTimeout, "shadow-timeout", cfg.ShadowTimeout, "Shadow read timeout") flag.StringVar(&cfg.SentryDSN, "sentry-dsn", cfg.SentryDSN, "Sentry DSN (empty = disabled)") flag.StringVar(&cfg.SentryEnv, "sentry-env", cfg.SentryEnv, "Sentry environment") diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index 65c9026f6..fa122a72c 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -44,6 +44,7 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-secondary-blocking-replay-queue-size` | `0` | Bounded queue for mutating blocking-command replays. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-mode` | `dual-write` | Proxy mode (see below) | | `-secondary-timeout` | `30s` | End-to-end secondary write deadline, including queue wait | +| `-secondary-script-timeout` | `5m` | End-to-end secondary Lua-script write deadline, including queue wait. `0` follows `-secondary-timeout` | | `-shadow-timeout` | `3s` | Shadow read timeout | | `-sentry-dsn` | (empty) | Sentry DSN (empty = disabled) | | `-sentry-env` | (empty) | Sentry environment name | @@ -101,6 +102,7 @@ docker run --rm \ -secondary-script-concurrency 2 \ -mode dual-write-shadow \ -secondary-timeout 30s \ + -secondary-script-timeout 5m \ -shadow-timeout 3s \ -sentry-dsn "${SENTRY_DSN}" \ -sentry-env production \ @@ -443,7 +445,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. - Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` at least `proxy replicas sharing one client IP * -elastickv-pool-size + 128`; PubSub and shadow PubSub use dedicated connections outside the command pool. With the HA compose defaults, use at least `512`. Keep `-secondary-write-concurrency` at or below the pool size. -- If script drops rise while backend pool waits stay at zero, the bottleneck is server-side Lua replay or wide-column cleanup, not connection acquisition. Keep `-secondary-script-concurrency` low and profile ElasticKV before raising it. +- If script drops rise while backend pool waits stay at zero, the bottleneck is server-side Lua replay or wide-column cleanup, not connection acquisition. Keep `-secondary-script-concurrency` low; use `-secondary-script-timeout` for burst backlog and profile ElasticKV before raising concurrency. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. ### High divergence count diff --git a/proxy/async_queue.go b/proxy/async_queue.go index 5edc85aa0..7807eec71 100644 --- a/proxy/async_queue.go +++ b/proxy/async_queue.go @@ -33,10 +33,14 @@ func (d *DualWriter) startAsyncDispatchers() { } func (d *DualWriter) enqueueAsync(queue chan asyncTask, slots chan struct{}, queueName string, fn func(context.Context)) { + d.enqueueAsyncWithTimeout(queue, slots, queueName, d.cfg.SecondaryTimeout, fn) +} + +func (d *DualWriter) enqueueAsyncWithTimeout(queue chan asyncTask, slots chan struct{}, queueName string, timeout time.Duration, fn func(context.Context)) { now := time.Now() task := asyncTask{enqueuedAt: now, fn: fn} - if d.cfg.SecondaryTimeout > 0 { - task.deadline = now.Add(d.cfg.SecondaryTimeout) + if timeout > 0 { + task.deadline = now.Add(timeout) } // Holding mu makes the closed check and channel send atomic with Close. diff --git a/proxy/config.go b/proxy/config.go index dfe47b724..e75aa0ab6 100644 --- a/proxy/config.go +++ b/proxy/config.go @@ -3,10 +3,11 @@ package proxy import "time" const ( - defaultSecondaryTimeout = 30 * time.Second - defaultShadowTimeout = 3 * time.Second - defaultPubSubCompareWindow = 2 * time.Second - defaultPubSubSweepInterval = 500 * time.Millisecond + defaultSecondaryTimeout = 30 * time.Second + defaultSecondaryScriptTimeout = 5 * time.Minute + defaultShadowTimeout = 3 * time.Second + defaultPubSubCompareWindow = 2 * time.Second + defaultPubSubSweepInterval = 500 * time.Millisecond ) // ProxyMode controls which backends receive reads and writes. @@ -60,6 +61,9 @@ type ProxyConfig struct { SecondaryPassword string Mode ProxyMode SecondaryTimeout time.Duration + // SecondaryScriptTimeout bounds queued and running Lua-script secondary + // writes. Zero falls back to SecondaryTimeout. + SecondaryScriptTimeout time.Duration // SecondaryWriteConcurrency limits all concurrent asynchronous secondary // writes, including Lua scripts. Zero keeps the package default. SecondaryWriteConcurrency int @@ -94,6 +98,7 @@ func DefaultConfig() ProxyConfig { SecondaryAddr: "localhost:6380", Mode: ModeDualWrite, SecondaryTimeout: defaultSecondaryTimeout, + SecondaryScriptTimeout: defaultSecondaryScriptTimeout, SecondaryBlockingReplayConcurrency: maxBlockingReplayGoroutines, ShadowTimeout: defaultShadowTimeout, SentrySampleRate: 1.0, diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index d6741a8f7..5f4972a00 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -188,7 +188,8 @@ func NewDualWriter(primary, secondary Backend, cfg ProxyConfig, metrics *ProxyMe "write_queue_capacity", cap(d.writeQueueSlots), "script_queue_capacity", cap(d.scriptQueueSlots), "blocking_replay_queue_capacity", cap(d.blockingReplayQueueSlots), - "timeout", cfg.SecondaryTimeout) + "timeout", cfg.SecondaryTimeout, + "script_timeout", secondaryScriptTimeout(cfg)) return d } @@ -822,7 +823,7 @@ func (d *DualWriter) goWrite(fn func(context.Context)) { // goScript launches fn in a bounded Lua-script write goroutine. // It uses a smaller class limit while also consuming the shared write limit. func (d *DualWriter) goScript(fn func(context.Context)) { - d.enqueueAsync(d.scriptQueue, d.scriptQueueSlots, asyncQueueScript, fn) + d.enqueueAsyncWithTimeout(d.scriptQueue, d.scriptQueueSlots, asyncQueueScript, secondaryScriptTimeout(d.cfg), fn) } // goBlockingReplay queues fn for bounded secondary replay of mutating blocking @@ -844,6 +845,13 @@ func (d *DualWriter) goAsync(fn func(context.Context)) { d.goWrite(fn) } +func secondaryScriptTimeout(cfg ProxyConfig) time.Duration { + if cfg.SecondaryScriptTimeout > 0 { + return cfg.SecondaryScriptTimeout + } + return cfg.SecondaryTimeout +} + func (d *DualWriter) startBackendPoolSampler() { backends := backendPoolSampleBackends(d.primary, d.secondary) if len(backends) == 0 { diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 623e7a4a6..8b39a4809 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -184,6 +184,7 @@ func TestDefaultConfig(t *testing.T) { assert.Equal(t, "localhost:6380", cfg.SecondaryAddr) assert.Equal(t, ModeDualWrite, cfg.Mode) assert.Equal(t, 30*time.Second, cfg.SecondaryTimeout) + assert.Equal(t, 5*time.Minute, cfg.SecondaryScriptTimeout) } // ========== compare.go tests ========== @@ -1148,6 +1149,47 @@ func TestDualWriter_QueuedWorkExpiresBeforeBackendDispatch(t *testing.T) { metrics.AsyncDropsByQueue.WithLabelValues(asyncQueueScript, asyncDropExpired)), 0.001) } +func TestDualWriter_ScriptQueueUsesScriptTimeout(t *testing.T) { + metrics := newTestMetrics() + d := NewDualWriter( + newMockBackend("primary"), + newMockBackend("secondary"), + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: 30 * time.Millisecond, + SecondaryScriptTimeout: 200 * time.Millisecond, + SecondaryWriteConcurrency: 1, + SecondaryScriptConcurrency: 1, + SecondaryScriptQueueCapacity: 1, + }, + metrics, + newTestSentry(), + testLogger, + ) + + blocker := make(chan struct{}) + started := make(chan struct{}) + d.goWrite(func(context.Context) { + close(started) + <-blocker + }) + <-started + + ran := make(chan struct{}) + d.goScript(func(context.Context) { close(ran) }) + time.Sleep(50 * time.Millisecond) + close(blocker) + select { + case <-ran: + case <-time.After(time.Second): + t.Fatal("script expired before secondary-script-timeout") + } + d.Close() + + assert.InDelta(t, 0, testutil.ToFloat64( + metrics.AsyncDropsByQueue.WithLabelValues(asyncQueueScript, asyncDropExpired)), 0.001) +} + func TestDualWriter_ExpiredQueuedWorkDoesNotWaitForSemaphore(t *testing.T) { metrics := newTestMetrics() d := &DualWriter{ From 040ed5a16b2427e7251ba827f7e8a2f91f943a34 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 12:23:29 +0900 Subject: [PATCH 04/18] redis-proxy: clarify script concurrency default --- cmd/redis-proxy/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index 108d77639..eae7cff95 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -55,7 +55,7 @@ func run() error { flag.IntVar(&primaryPoolSize, "primary-pool-size", primaryPoolSize, "Primary Redis backend connection pool size") flag.IntVar(&elasticKVPoolSize, "elastickv-pool-size", elasticKVPoolSize, "ElasticKV backend connection pool size") flag.IntVar(&secondaryWriteConcurrency, "secondary-write-concurrency", secondaryWriteConcurrency, "Maximum concurrent asynchronous secondary writes including scripts (0 = half of secondary backend pool size)") - flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = secondary write concurrency / 32, capped at 2)") + flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = secondary write concurrency / 32, minimum 1, capped at 2)") flag.IntVar(&secondaryBlockingReplayConcurrency, "secondary-blocking-replay-concurrency", secondaryBlockingReplayConcurrency, "Maximum concurrent asynchronous secondary mutating blocking-command replays (0 = capped remaining secondary backend pool capacity after writes)") flag.IntVar(&secondaryWriteQueueSize, "secondary-write-queue-size", secondaryWriteQueueSize, "Maximum queued asynchronous secondary writes (0 = derived from write concurrency)") flag.IntVar(&secondaryScriptQueueSize, "secondary-script-queue-size", secondaryScriptQueueSize, "Maximum queued asynchronous secondary Lua-script writes (0 = derived from script concurrency)") From 68af55e03898b3f29e673bc5ddd218f2de5a340c Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 12:26:50 +0900 Subject: [PATCH 05/18] redis-proxy: align script concurrency with runtime --- README.md | 2 +- cmd/redis-proxy/main.go | 4 ++-- cmd/redis-proxy/main_test.go | 2 +- docs/redis-proxy-deployment.md | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 26a9f233a..788f26108 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ docker run --rm \ -secondary elastickv.internal:6380 \ -elastickv-pool-size 192 \ -secondary-write-concurrency 96 \ - -secondary-script-concurrency 2 \ + -secondary-script-concurrency 3 \ -mode dual-write ``` diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index eae7cff95..7c95c0bf1 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -23,7 +23,7 @@ const ( metricsShutdownTimeout = 5 * time.Second secondaryWriteConcurrencyDivisor = 2 secondaryScriptConcurrencyDivisor = 32 - secondaryScriptConcurrencyCap = 2 + secondaryScriptConcurrencyCap = 3 ) func main() { @@ -55,7 +55,7 @@ func run() error { flag.IntVar(&primaryPoolSize, "primary-pool-size", primaryPoolSize, "Primary Redis backend connection pool size") flag.IntVar(&elasticKVPoolSize, "elastickv-pool-size", elasticKVPoolSize, "ElasticKV backend connection pool size") flag.IntVar(&secondaryWriteConcurrency, "secondary-write-concurrency", secondaryWriteConcurrency, "Maximum concurrent asynchronous secondary writes including scripts (0 = half of secondary backend pool size)") - flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = secondary write concurrency / 32, minimum 1, capped at 2)") + flag.IntVar(&secondaryScriptConcurrency, "secondary-script-concurrency", secondaryScriptConcurrency, "Maximum concurrent asynchronous secondary Lua-script writes within the write limit (0 = secondary write concurrency / 32, minimum 1, capped at 3)") flag.IntVar(&secondaryBlockingReplayConcurrency, "secondary-blocking-replay-concurrency", secondaryBlockingReplayConcurrency, "Maximum concurrent asynchronous secondary mutating blocking-command replays (0 = capped remaining secondary backend pool capacity after writes)") flag.IntVar(&secondaryWriteQueueSize, "secondary-write-queue-size", secondaryWriteQueueSize, "Maximum queued asynchronous secondary writes (0 = derived from write concurrency)") flag.IntVar(&secondaryScriptQueueSize, "secondary-script-queue-size", secondaryScriptQueueSize, "Maximum queued asynchronous secondary Lua-script writes (0 = derived from script concurrency)") diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 7e566f3f7..76e7c8118 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -168,7 +168,7 @@ func TestDeriveSecondaryConcurrencyFromDefaultElasticKVPool(t *testing.T) { require.Equal(t, 192, poolSize) require.Equal(t, 96, writeConcurrency) - require.Equal(t, 2, scriptConcurrency) + require.Equal(t, 3, scriptConcurrency) require.Equal(t, 32, blockingConcurrency) require.NoError(t, validateSecondaryConcurrency( proxy.ModeDualWrite, diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index fa122a72c..3c42d599c 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -37,7 +37,7 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-primary-pool-size` | `128` | Primary Redis backend connection pool size | | `-elastickv-pool-size` | `192` | ElasticKV backend connection pool size | | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` | -| `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives one thirty-second of the shared write limit, capped at `2`, minimum `1` | +| `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives one thirty-second of the shared write limit, capped at `3`, minimum `1` | | `-secondary-blocking-replay-concurrency` | `0` | Mutating blocking-command replay sublimit. `0` uses remaining secondary backend pool capacity, capped at `32` | | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-secondary-script-queue-size` | `0` | Bounded queue for secondary Lua-script writes. `0` derives `64 * concurrency`, clamped to `64..8192` | @@ -99,7 +99,7 @@ docker run --rm \ -secondary elastickv.internal:6380 \ -elastickv-pool-size 192 \ -secondary-write-concurrency 96 \ - -secondary-script-concurrency 2 \ + -secondary-script-concurrency 3 \ -mode dual-write-shadow \ -secondary-timeout 30s \ -secondary-script-timeout 5m \ @@ -124,7 +124,7 @@ services: - -secondary=elastickv:6380 - -elastickv-pool-size=192 - -secondary-write-concurrency=96 - - -secondary-script-concurrency=2 + - -secondary-script-concurrency=3 - -mode=dual-write-shadow - -metrics=:9191 depends_on: From f3a47b669d232dbb54f1b43de7dffebf8020f716 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 12:28:35 +0900 Subject: [PATCH 06/18] redis-proxy: extend ElasticKV read deadline --- adapter/redis_lua.go | 2 +- docs/redis-proxy-deployment.md | 2 +- proxy/backend.go | 16 +++++++++------- proxy/proxy_test.go | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/adapter/redis_lua.go b/adapter/redis_lua.go index d6792e937..48f05e224 100644 --- a/adapter/redis_lua.go +++ b/adapter/redis_lua.go @@ -833,7 +833,7 @@ func (r *RedisServer) execLuaCompat(conn redcon.Conn, command string, args [][]b stringArgs[i] = string(arg) } - ctx, cancel := context.WithTimeout(context.Background(), redisLuaDispatchTimeout) + ctx, cancel := context.WithTimeout(context.Background(), redisDispatchTimeout) defer cancel() var reply luaReply diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index 3c42d599c..c5a3e0a7b 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -421,7 +421,7 @@ groups: | ElasticKV connection pool size | 192 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections | | ElasticKV Redis per-peer connection cap | 512 | Default server-side cap: two proxy replicas at pool `192`, plus dedicated PubSub/shadow PubSub headroom | | Dial timeout | 5s | Backend connection timeout | -| Read timeout | 3s | Backend read timeout | +| Read timeout | Redis: 3s, ElasticKV: 35s | Backend read timeout | | Write timeout | 3s | Backend write timeout | | Async write concurrency fallback | 4096 | Package fallback; the command derives a lower limit from backend pool size | | Shadow read goroutine limit | 1024 | Max concurrent shadow comparisons | diff --git a/proxy/backend.go b/proxy/backend.go index bd9394a93..a15224f74 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -10,13 +10,14 @@ import ( ) const ( - defaultPoolSize = 128 - defaultElasticKVPoolSize = 192 - defaultDialTimeout = 5 * time.Second - defaultReadTimeout = 3 * time.Second - defaultWriteTimeout = 3 * time.Second - blockingReadGrace = 10 * time.Second - respProtocolV2 = 2 + defaultPoolSize = 128 + defaultElasticKVPoolSize = 192 + defaultDialTimeout = 5 * time.Second + defaultReadTimeout = 3 * time.Second + defaultElasticKVReadTimeout = 35 * time.Second + defaultWriteTimeout = 3 * time.Second + blockingReadGrace = 10 * time.Second + respProtocolV2 = 2 ) // Backend abstracts a Redis-protocol endpoint (real Redis or ElasticKV). @@ -81,6 +82,7 @@ func DefaultBackendOptions() BackendOptions { func DefaultElasticKVBackendOptions() BackendOptions { opts := DefaultBackendOptions() opts.PoolSize = defaultElasticKVPoolSize + opts.ReadTimeout = defaultElasticKVReadTimeout return opts } diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 8b39a4809..68abbdc18 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -1379,7 +1379,7 @@ func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() assert.Equal(t, 192, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) - assert.Equal(t, 3*time.Second, opts.ReadTimeout) + assert.Equal(t, 35*time.Second, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout) } From a802b7dbabc1cfee324eb456337890fd13cfae25 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 12:40:48 +0900 Subject: [PATCH 07/18] redis-proxy: preserve HLC renewal under load --- docs/architecture_overview.md | 2 +- kv/coordinator.go | 26 +++++++--- kv/sharded_coordinator.go | 2 +- main.go | 30 ++++++++++++ main_encryption_rotate_on_startup_test.go | 59 ++++++++++++++++++++++- monitoring/hlc.go | 10 ++-- monitoring/hlc_test.go | 2 +- 7 files changed, 113 insertions(+), 18 deletions(-) diff --git a/docs/architecture_overview.md b/docs/architecture_overview.md index 796d3d7fe..063d34b90 100644 --- a/docs/architecture_overview.md +++ b/docs/architecture_overview.md @@ -170,7 +170,7 @@ sequenceDiagram participant H as "HLC (all nodes)" participant Tx as "Txn / MVCC read-write path" - loop "every hlcRenewalInterval (<3s)" + loop "every hlcRenewalInterval (<10s)" L->>RG: "Propose HLC lease (now + hlcPhysicalWindowMs)" RG-->>F: "Apply HLC lease entry" F->>H: "SetPhysicalCeiling(ms)" diff --git a/kv/coordinator.go b/kv/coordinator.go index 093d77686..15d5268e1 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -36,16 +36,25 @@ const dispatchLeaderRetryBudget = 5 * time.Second const dispatchLeaderRetryInterval = 25 * time.Millisecond // hlcPhysicalWindowMs is the duration in milliseconds that the Raft-agreed -// physical ceiling extends ahead of the current wall clock. Modelled after -// TiDB's TSO 3-second window: the leader commits ceiling = now + window, and -// renews before the window expires. A new leader inherits the committed ceiling -// so it never issues timestamps that collide with the previous leader's window. -const hlcPhysicalWindowMs int64 = 3_000 +// physical ceiling extends ahead of the current wall clock. The leader commits +// ceiling = now + window, and renews before the window expires. A new leader +// inherits the committed ceiling so it never issues timestamps that collide with +// the previous leader's window. Keep this comfortably above the renewal proposal +// timeout so a renewal that spends a few seconds queued in Raft still lands with +// a future ceiling. +const hlcPhysicalWindowMs int64 = 10_000 // hlcRenewalInterval controls how often the leader proposes a new ceiling. // Must be less than hlcPhysicalWindowMs to guarantee the window never expires. const hlcRenewalInterval = 1 * time.Second +// hlcRenewalProposalTimeout bounds a single HLC lease-renewal proposal. This is +// intentionally longer than hlcRenewalInterval: under write-heavy Redis proxy +// traffic, a renewal can sit behind normal Raft proposals for more than one +// second, and timing it out there causes avoidable fail-closed timestamp +// refusals. +const hlcRenewalProposalTimeout = dispatchLeaderRetryBudget + // CoordinatorOption is a functional option for Coordinate constructors. type CoordinatorOption func(*Coordinate) @@ -855,10 +864,11 @@ func (c *Coordinate) extendLeaseAfterRenewal(dispatchStart monoclock.Instant, ex // RunHLCLeaseRenewal runs a background loop that periodically proposes a new // physical ceiling to the Raft cluster while this node is the leader. // -// The ceiling is set to now + hlcPhysicalWindowMs (3 s) and is renewed every +// The ceiling is set to now + hlcPhysicalWindowMs (10 s) and is renewed every // hlcRenewalInterval (1 s), mirroring TiDB's TSO window strategy. Because the -// window is always at least 2 s ahead of any real timestamp, a new leader will -// never issue timestamps that overlap with the previous leader's window. +// window is always well ahead of any real timestamp during healthy renewal, a +// new leader will never issue timestamps that overlap with the previous +// leader's window. // // RunHLCLeaseRenewal blocks until ctx is cancelled; call it in a goroutine. func (c *Coordinate) RunHLCLeaseRenewal(ctx context.Context) { diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 8c6a81ad4..b3db2931e 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -2359,7 +2359,7 @@ func (c *ShardedCoordinator) renewHLCLeases(ctx context.Context) <-chan struct{} go func(gid uint64, group *ShardGroup) { defer wg.Done() defer c.finishHLCLeaseRenewal(gid) - pctx, cancel := context.WithTimeout(ctx, hlcRenewalInterval) + pctx, cancel := context.WithTimeout(ctx, hlcRenewalProposalTimeout) defer cancel() c.renewHLCLease(pctx, gid, group) }(gid, group) diff --git a/main.go b/main.go index dd53cabf0..a5a7cec9e 100644 --- a/main.go +++ b/main.go @@ -2196,6 +2196,8 @@ var _ kv.LeaseReadableCoordinator = (*startupGatedCoordinator)(nil) var _ kv.AllGroupsLeaseReadableCoordinator = (*startupGatedCoordinator)(nil) var _ kv.GroupRoutableCoordinator = (*startupGatedCoordinator)(nil) var _ kv.RaftMembershipCoordinator = (*startupGatedCoordinator)(nil) +var _ kv.TimestampAllocator = (*startupGatedCoordinator)(nil) +var _ kv.TimestampAfterAllocator = (*startupGatedCoordinator)(nil) func (c startupGatedCoordinator) Dispatch(ctx context.Context, reqs *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { if c.gate != nil && c.gate.blocked() { @@ -2236,6 +2238,34 @@ func (c startupGatedCoordinator) Clock() *kv.HLC { return c.inner.Clock() } +func (c startupGatedCoordinator) Next(ctx context.Context) (uint64, error) { + if alloc, ok := c.inner.(kv.TimestampAllocator); ok { + ts, err := alloc.Next(ctx) + return ts, errors.WithStack(err) + } + ts, err := kv.NextTimestampThrough(ctx, c.inner, "startup-gated timestamp") + return ts, errors.WithStack(err) +} + +func (c startupGatedCoordinator) NextAfter(ctx context.Context, min uint64) (uint64, error) { + if alloc, ok := c.inner.(kv.TimestampAfterAllocator); ok { + ts, err := alloc.NextAfter(ctx, min) + return ts, errors.WithStack(err) + } + ts, err := kv.NextTimestampAfterThrough(ctx, c.inner, min, "startup-gated timestamp after observed ts") + return ts, errors.WithStack(err) +} + +func (c startupGatedCoordinator) RecoverHLCLease(ctx context.Context) error { + type hlcLeaseRecoverer interface { + RecoverHLCLease(context.Context) error + } + if recoverer, ok := c.inner.(hlcLeaseRecoverer); ok { + return errors.WithStack(recoverer.RecoverHLCLease(ctx)) + } + return errors.New("startup-gated coordinator: hlc lease recovery unavailable") +} + func (c startupGatedCoordinator) LeaseRead(ctx context.Context) (uint64, error) { return kv.LeaseReadThrough(c.inner, ctx) //nolint:wrapcheck // Pass through coordinator errors unchanged. } diff --git a/main_encryption_rotate_on_startup_test.go b/main_encryption_rotate_on_startup_test.go index 9d8063deb..48b4adc9d 100644 --- a/main_encryption_rotate_on_startup_test.go +++ b/main_encryption_rotate_on_startup_test.go @@ -161,8 +161,14 @@ func (r *raceFireRotateStartupController) State() raftengine.State { } type stubStartupCoordinator struct { - dispatches atomic.Int32 - clock *kv.HLC + dispatches atomic.Int32 + nextCalls atomic.Int32 + nextAfter atomic.Int32 + recoverCalls atomic.Int32 + clock *kv.HLC + nextErr error + nextAfterErr error + recoverErr error } func (s *stubStartupCoordinator) Dispatch(context.Context, *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { @@ -191,6 +197,21 @@ func (s *stubStartupCoordinator) Clock() *kv.HLC { return s.clock } +func (s *stubStartupCoordinator) Next(context.Context) (uint64, error) { + s.nextCalls.Add(1) + return 101, s.nextErr +} + +func (s *stubStartupCoordinator) NextAfter(_ context.Context, min uint64) (uint64, error) { + s.nextAfter.Add(1) + return min + 1, s.nextAfterErr +} + +func (s *stubStartupCoordinator) RecoverHLCLease(context.Context) error { + s.recoverCalls.Add(1) + return s.recoverErr +} + func TestInstallEncryptionRotateOnStartupRequest_ImmediateLeaderWaitRuns(t *testing.T) { t.Parallel() controller := &fakeRotateStartupController{state: raftengine.StateLeader} @@ -602,6 +623,40 @@ func TestStartupGatedCoordinator_BlocksAdapterDispatchDuringAsyncRotation(t *tes } } +func TestStartupGatedCoordinator_ForwardsTimestampCapabilities(t *testing.T) { + t.Parallel() + sentinel := errors.New("recover failed") + inner := &stubStartupCoordinator{clock: kv.NewHLC(), recoverErr: sentinel} + coord := startupGatedCoordinator{ + inner: inner, + gate: &startupPublicKVGate{blockMutator: func() bool { return true }}, + } + + ts, err := coord.Next(context.Background()) + if err != nil || ts != 101 { + t.Fatalf("Next() = (%d, %v), want (101, nil)", ts, err) + } + next, err := coord.NextAfter(context.Background(), 200) + if err != nil || next != 201 { + t.Fatalf("NextAfter() = (%d, %v), want (201, nil)", next, err) + } + if err := coord.RecoverHLCLease(context.Background()); !errors.Is(err, sentinel) { + t.Fatalf("RecoverHLCLease() err=%v, want sentinel", err) + } + if got := inner.nextCalls.Load(); got != 1 { + t.Fatalf("inner Next calls=%d, want 1", got) + } + if got := inner.nextAfter.Load(); got != 1 { + t.Fatalf("inner NextAfter calls=%d, want 1", got) + } + if got := inner.recoverCalls.Load(); got != 1 { + t.Fatalf("inner RecoverHLCLease calls=%d, want 1", got) + } + if got := inner.dispatches.Load(); got != 0 { + t.Fatalf("timestamp capability forwarding called Dispatch %d times, want 0", got) + } +} + func TestRuntimeServerRunner_PrepareAdminForwardServersDoesNotBindPublicListeners(t *testing.T) { t.Parallel() ctx := context.Background() diff --git a/monitoring/hlc.go b/monitoring/hlc.go index 2b7dfb9b9..08c017602 100644 --- a/monitoring/hlc.go +++ b/monitoring/hlc.go @@ -40,7 +40,7 @@ type HLCSource interface { type HLCMetrics struct { // physicalCeilingSeconds is the Raft-agreed physical ceiling, // expressed as a Unix timestamp in seconds. A leader's - // ceiling should sit ~hlcPhysicalWindowMs ms (3 s default) + // ceiling should sit ~hlcPhysicalWindowMs ms (10 s default) // ahead of wall_now; a stagnant value means the lease // renewer has stopped landing entries. physicalCeilingSeconds prometheus.Gauge @@ -49,7 +49,7 @@ type HLCMetrics struct { // (healthy, with the leader's renewer keeping up). Zero or // positive means the ceiling has expired locally — // NextFenced is at-or-past the rejection boundary. Alert - // on wallSkewSeconds > -0.5 (half of hlcPhysicalWindowMs) + // on wallSkewSeconds > -5 (half of hlcPhysicalWindowMs) // to catch a renewal stall before the fence trips. wallSkewSeconds prometheus.Gauge // nextFencedRejectionsTotal is the cumulative number of @@ -68,7 +68,7 @@ func newHLCMetrics(registerer prometheus.Registerer) *HLCMetrics { }), wallSkewSeconds: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "elastickv_hlc_wall_skew_seconds", - Help: "wall_now - physicalCeiling, in seconds. Negative is healthy (ceiling is in the future); zero or positive means NextFenced is at-or-past the rejection boundary. Alert on > -0.5 to catch renewal stalls before the HLC-4 (iii) fence trips.", + Help: "wall_now - physicalCeiling, in seconds. Negative is healthy (ceiling is in the future); zero or positive means NextFenced is at-or-past the rejection boundary. Alert on > -5 to catch renewal stalls before the HLC-4 (iii) fence trips.", }), nextFencedRejectionsTotal: prometheus.NewCounter(prometheus.CounterOpts{ Name: "elastickv_hlc_next_fenced_rejections_total", @@ -146,12 +146,12 @@ func (o *HLCObserver) observeOnce(source HLCSource) { o.metrics.physicalCeilingSeconds.Set(float64(ceilingMs) / hlcMsPerSecond) // Pre-bootstrap (ceilingMs == 0) the skew computation would produce // nowMs / 1000 ≈ 1.75e9 seconds (~55 years), which would - // immediately trip the recommended `wallSkewSeconds > -0.5` alert + // immediately trip the recommended `wallSkewSeconds > -5` alert // on every cold-start node until the first HLC lease lands. Hold // the gauge at 0 in that case — the physicalCeilingSeconds gauge // already exposes the pre-bootstrap state (also 0), so an alert // of the form - // `wallSkewSeconds > -0.5 AND physicalCeilingSeconds > 0` + // `wallSkewSeconds > -5 AND physicalCeilingSeconds > 0` // behaves correctly. (Claude review medium on PR #879.) if ceilingMs > 0 { o.metrics.wallSkewSeconds.Set(float64(nowMs-ceilingMs) / hlcMsPerSecond) diff --git a/monitoring/hlc_test.go b/monitoring/hlc_test.go index 644b798d4..279c6a6e6 100644 --- a/monitoring/hlc_test.go +++ b/monitoring/hlc_test.go @@ -104,7 +104,7 @@ func TestHLCObserver_NilSourceIsNoop(t *testing.T) { // skew gauge stays at 0 when PhysicalCeiling() == 0 (pre-bootstrap, // before the first HLC lease lands). Without this guard the // observer would publish wallSkewSeconds ≈ 1.75e9 (~55 years) and -// trip the recommended `wallSkewSeconds > -0.5` alert on every cold +// trip the recommended `wallSkewSeconds > -5` alert on every cold // start until the first lease propagates. Regression for the medium // finding on PR #879. func TestHLCObserveOnce_PreBootstrapHoldsSkewAtZero(t *testing.T) { From 9dc4adf7adf5929b399a443c3b160bcb645eaef3 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 13:24:39 +0900 Subject: [PATCH 08/18] redis-proxy: avoid Lua rewrite cleanup scans --- adapter/redis_compat_helpers.go | 106 ++++++++++++++++++++++++++++++++ adapter/redis_lua_context.go | 17 +++-- adapter/redis_txn_test.go | 104 +++++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 4 deletions(-) diff --git a/adapter/redis_compat_helpers.go b/adapter/redis_compat_helpers.go index df4c0caa8..016b423c6 100644 --- a/adapter/redis_compat_helpers.go +++ b/adapter/redis_compat_helpers.go @@ -1112,6 +1112,112 @@ func (r *RedisServer) deleteLogicalKeyElems(ctx context.Context, key []byte, rea return elems, existed, nil } +func (r *RedisServer) deleteLogicalKeyElemsForType(ctx context.Context, key []byte, readTS uint64, typ redisValueType) ([]*kv.Elem[kv.OP], bool, error) { + switch typ { + case redisTypeNone: + return nil, false, nil + case redisTypeString: + elems, err := r.deleteStringLikeElems(ctx, key, readTS) + return elems, true, err + case redisTypeList: + return r.deleteListLogicalKeyElems(ctx, key, readTS) + case redisTypeHash: + return r.deleteHashLogicalKeyElems(ctx, key, readTS) + case redisTypeSet: + return r.deleteSetLogicalKeyElems(ctx, key, readTS) + case redisTypeZSet: + return r.deleteZSetLogicalKeyElems(ctx, key, readTS) + case redisTypeStream: + return r.deleteStreamLogicalKeyElems(ctx, key, readTS) + } + return nil, false, errors.WithStack(errors.AssertionFailedf("unknown redis type %v", typ)) +} + +func (r *RedisServer) deleteStringLikeElems(ctx context.Context, key []byte, readTS uint64) ([]*kv.Elem[kv.OP], error) { + var elems []*kv.Elem[kv.OP] + for _, internalKey := range [][]byte{ + redisStrKey(key), + key, // legacy bare string key + redisHLLKey(key), + redisTTLKey(key), + } { + ok, err := r.store.ExistsAt(ctx, internalKey, readTS) + if err != nil { + return nil, errors.WithStack(err) + } + if ok { + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: internalKey}) + } + } + return elems, nil +} + +func (r *RedisServer) deleteListLogicalKeyElems(ctx context.Context, key []byte, readTS uint64) ([]*kv.Elem[kv.OP], bool, error) { + elems, err := r.deleteStringLikeElems(ctx, key, readTS) + if err != nil { + return nil, false, err + } + listElems, err := r.deleteListElems(ctx, key, readTS) + if err != nil { + return nil, false, err + } + return append(elems, listElems...), true, nil +} + +func (r *RedisServer) deleteHashLogicalKeyElems(ctx context.Context, key []byte, readTS uint64) ([]*kv.Elem[kv.OP], bool, error) { + elems, err := r.deleteStringLikeElems(ctx, key, readTS) + if err != nil { + return nil, false, err + } + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: redisHashKey(key)}) + hashElems, err := r.deleteWideColumnElems(ctx, readTS, + store.HashFieldScanPrefix(key), store.HashMetaKey(key), store.HashMetaDeltaScanPrefix(key)) + if err != nil { + return nil, false, err + } + return append(elems, hashElems...), true, nil +} + +func (r *RedisServer) deleteSetLogicalKeyElems(ctx context.Context, key []byte, readTS uint64) ([]*kv.Elem[kv.OP], bool, error) { + elems, err := r.deleteStringLikeElems(ctx, key, readTS) + if err != nil { + return nil, false, err + } + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: redisSetKey(key)}) + setElems, err := r.deleteWideColumnElems(ctx, readTS, + store.SetMemberScanPrefix(key), store.SetMetaKey(key), store.SetMetaDeltaScanPrefix(key)) + if err != nil { + return nil, false, err + } + return append(elems, setElems...), true, nil +} + +func (r *RedisServer) deleteZSetLogicalKeyElems(ctx context.Context, key []byte, readTS uint64) ([]*kv.Elem[kv.OP], bool, error) { + elems, err := r.deleteStringLikeElems(ctx, key, readTS) + if err != nil { + return nil, false, err + } + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: redisZSetKey(key)}) + zsetElems, err := r.deleteZSetWideColumnElems(ctx, key, readTS) + if err != nil { + return nil, false, err + } + return append(elems, zsetElems...), true, nil +} + +func (r *RedisServer) deleteStreamLogicalKeyElems(ctx context.Context, key []byte, readTS uint64) ([]*kv.Elem[kv.OP], bool, error) { + elems, err := r.deleteStringLikeElems(ctx, key, readTS) + if err != nil { + return nil, false, err + } + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: redisStreamKey(key)}) + streamElems, err := r.deleteStreamWideColumnElems(ctx, key, readTS) + if err != nil { + return nil, false, err + } + return append(elems, streamElems...), true, nil +} + // deleteStreamWideColumnElems returns delete operations for all stream // wide-column keys: the meta key (if it exists) and every entry under the // entry scan prefix. Total results are capped at maxWideColumnItems to diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index dd35b9dfa..3267ef974 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -3658,18 +3658,27 @@ func (c *luaScriptContext) commitPlanForKey(ctx context.Context, key string, com return luaKeyPlan{}, err } - startType, err := c.server.keyTypeAt(ctx, []byte(key), c.startTS) + keyBytes := []byte(key) + rawStartType, err := c.server.rawKeyTypeAt(ctx, keyBytes, c.startTS) + if err != nil { + return luaKeyPlan{}, err + } + startType, err := c.server.applyTTLFilter(ctx, keyBytes, c.startTS, rawStartType) if err != nil { return luaKeyPlan{}, err } var deleteElems []*kv.Elem[kv.OP] - readKeys := luaWideFenceReadKeysForPlan([]byte(key), finalType, startType, valuePlan.preserveExisting) + readKeys := luaWideFenceReadKeysForPlan(keyBytes, finalType, startType, valuePlan.preserveExisting) if !valuePlan.preserveExisting { - deleteElems, _, err = c.server.deleteLogicalKeyElems(ctx, []byte(key), c.startTS) + if c.everDeleted[key] && rawStartType != redisTypeNone { + deleteElems, _, err = c.server.deleteLogicalKeyElems(ctx, keyBytes, c.startTS) + } else { + deleteElems, _, err = c.server.deleteLogicalKeyElemsForType(ctx, keyBytes, c.startTS, rawStartType) + } if err != nil { return luaKeyPlan{}, err } - deleteElems = append(deleteElems, redisTxnWideCollectionFenceElems([]byte(key))...) + deleteElems = append(deleteElems, redisTxnWideCollectionFenceElems(keyBytes)...) } dataElems := make([]*kv.Elem[kv.OP], 0, len(deleteElems)+len(valuePlan.elems)) diff --git a/adapter/redis_txn_test.go b/adapter/redis_txn_test.go index a44bad0b0..6eb0088c8 100644 --- a/adapter/redis_txn_test.go +++ b/adapter/redis_txn_test.go @@ -639,6 +639,110 @@ func TestLuaWideFenceReadKeysForPlan(t *testing.T) { require.Nil(t, luaWideFenceReadKeysForPlan(key, redisTypeString, redisTypeString, true)) } +type luaCleanupScanTrackingStore struct { + store.MVCCStore + fullScanStarts [][]byte +} + +func (s *luaCleanupScanTrackingStore) ScanAt(ctx context.Context, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + if limit == store.MaxDeltaScanLimit { + s.fullScanStarts = append(s.fullScanStarts, bytes.Clone(start)) + } + return s.MVCCStore.ScanAt(ctx, start, end, limit, ts) +} + +func newLuaCommitPlanTestContext(server *RedisServer, startTS uint64) *luaScriptContext { + return &luaScriptContext{ + server: server, + startTS: startTS, + touched: map[string]struct{}{}, + readKeys: map[string][]byte{}, + deleted: map[string]bool{}, + everDeleted: map[string]bool{}, + negativeType: map[string]bool{}, + strings: map[string]*luaStringState{}, + lists: map[string]*luaListState{}, + hashes: map[string]*luaHashState{}, + sets: map[string]*luaSetState{}, + zsets: map[string]*luaZSetState{}, + streams: map[string]*luaStreamState{}, + ttls: map[string]*luaTTLState{}, + } +} + +func TestLuaCommitPlanForAbsentRewriteSkipsFullLogicalCleanupScans(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base := store.NewMVCCStore() + tracking := &luaCleanupScanTrackingStore{MVCCStore: base} + server := NewRedisServer(nil, "", tracking, newLocalAdapterCoordinator(base), nil, nil) + key := "lua:absent-rewrite" + + scriptCtx := newLuaCommitPlanTestContext(server, 10) + scriptCtx.strings[key] = &luaStringState{loaded: true, exists: true, dirty: true, value: []byte("v")} + scriptCtx.ttls[key] = &luaTTLState{loaded: true} + + plan, err := scriptCtx.commitPlanForKey(ctx, key, 11) + require.NoError(t, err) + require.Empty(t, tracking.fullScanStarts) + require.True(t, elemKeysContain(plan.elems, redisStrKey([]byte(key)))) + require.True(t, elemKeysContain(plan.elems, redisTxnWideHashFenceKey([]byte(key)))) +} + +func TestLuaCommitPlanForExistingListRewriteOnlyScansListCleanup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base := store.NewMVCCStore() + tracking := &luaCleanupScanTrackingStore{MVCCStore: base} + server := NewRedisServer(nil, "", tracking, newLocalAdapterCoordinator(base), nil, nil) + key := []byte("lua:list-rewrite") + keyString := string(key) + meta, err := store.MarshalListMeta(store.ListMeta{Len: 1}) + require.NoError(t, err) + require.NoError(t, base.PutAt(ctx, store.ListMetaKey(key), meta, 10, 0)) + require.NoError(t, base.PutAt(ctx, listItemKey(key, 0), []byte("old"), 10, 0)) + + scriptCtx := newLuaCommitPlanTestContext(server, 11) + scriptCtx.lists[keyString] = &luaListState{ + loaded: true, + exists: true, + dirty: true, + materialized: true, + values: []string{"new"}, + } + scriptCtx.ttls[keyString] = &luaTTLState{loaded: true} + + _, err = scriptCtx.commitPlanForKey(ctx, keyString, 12) + require.NoError(t, err) + requireScanStartsIncludePrefix(t, tracking.fullScanStarts, append(append([]byte(nil), []byte(store.ListItemPrefix)...), key...)) + requireScanStartsIncludePrefix(t, tracking.fullScanStarts, store.ListMetaDeltaScanPrefix(key)) + requireScanStartsIncludePrefix(t, tracking.fullScanStarts, store.ListClaimScanPrefix(key)) + requireScanStartsExcludePrefix(t, tracking.fullScanStarts, store.HashFieldScanPrefix(key)) + requireScanStartsExcludePrefix(t, tracking.fullScanStarts, store.SetMemberScanPrefix(key)) + requireScanStartsExcludePrefix(t, tracking.fullScanStarts, store.ZSetMemberScanPrefix(key)) + requireScanStartsExcludePrefix(t, tracking.fullScanStarts, store.ZSetScoreScanPrefix(key)) + requireScanStartsExcludePrefix(t, tracking.fullScanStarts, store.StreamEntryScanPrefix(key)) +} + +func requireScanStartsIncludePrefix(t *testing.T, starts [][]byte, prefix []byte) { + t.Helper() + for _, start := range starts { + if bytes.HasPrefix(start, prefix) { + return + } + } + t.Fatalf("expected a scan under prefix %q, got %q", prefix, starts) +} + +func requireScanStartsExcludePrefix(t *testing.T, starts [][]byte, prefix []byte) { + t.Helper() + for _, start := range starts { + require.Falsef(t, bytes.HasPrefix(start, prefix), "unexpected scan under prefix %q in %q", prefix, starts) + } +} + func TestRedisTxnSetReplacementConflictsWithConcurrentWideHashWrite(t *testing.T) { t.Parallel() From 486037c994bc131dd6c819c6cf6ce65c654833d3 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 13:29:09 +0900 Subject: [PATCH 09/18] redis-proxy: align backend timeout budget --- adapter/redis.go | 9 +++++---- docs/redis-proxy-deployment.md | 2 +- proxy/backend.go | 9 +++++---- proxy/proxy_test.go | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/adapter/redis.go b/adapter/redis.go index 87dd78dcb..e503d860d 100644 --- a/adapter/redis.go +++ b/adapter/redis.go @@ -115,11 +115,12 @@ const ( ) const ( - redisDispatchTimeout = 10 * time.Second + redisDispatchTimeout = 30 * time.Second // redisLuaDispatchTimeout gives EVAL/EVALSHA enough room for migration - // scripts that expand into thousands of Redis calls while keeping regular - // commands on the tighter dispatch deadline. - redisLuaDispatchTimeout = 30 * time.Second + // scripts that expand into thousands of Redis calls. Regular commands stay + // on redisDispatchTimeout so non-script heavy paths cannot hold worker slots + // for the full script replay budget. + redisLuaDispatchTimeout = 5 * time.Minute // defaultRedisBlockWaitFallback is the safety-net poll interval for // blocking-command wait loops when no in-process write signal arrives. // Signals cover normal XADD / ZADD / ZINCRBY wakeups immediately; this diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index c5a3e0a7b..b35b66d4c 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -421,7 +421,7 @@ groups: | ElasticKV connection pool size | 192 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections | | ElasticKV Redis per-peer connection cap | 512 | Default server-side cap: two proxy replicas at pool `192`, plus dedicated PubSub/shadow PubSub headroom | | Dial timeout | 5s | Backend connection timeout | -| Read timeout | Redis: 3s, ElasticKV: 35s | Backend read timeout | +| Read timeout | Redis: 3s, ElasticKV: 5m10s | Backend read timeout; ElasticKV keeps 10s grace beyond `-secondary-script-timeout` | | Write timeout | 3s | Backend write timeout | | Async write concurrency fallback | 4096 | Package fallback; the command derives a lower limit from backend pool size | | Shadow read goroutine limit | 1024 | Max concurrent shadow comparisons | diff --git a/proxy/backend.go b/proxy/backend.go index a15224f74..c47b935a2 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -14,9 +14,9 @@ const ( defaultElasticKVPoolSize = 192 defaultDialTimeout = 5 * time.Second defaultReadTimeout = 3 * time.Second - defaultElasticKVReadTimeout = 35 * time.Second - defaultWriteTimeout = 3 * time.Second blockingReadGrace = 10 * time.Second + defaultElasticKVReadTimeout = defaultSecondaryScriptTimeout + blockingReadGrace + defaultWriteTimeout = 3 * time.Second respProtocolV2 = 2 ) @@ -77,8 +77,9 @@ func DefaultBackendOptions() BackendOptions { // should run the cluster with ELASTICKV_REDIS_PER_PEER_CONNECTIONS sized for // every proxy replica that may share one client IP, plus dedicated PubSub and // shadow PubSub connections outside this command pool. Lower the proxy pool -// instead for clusters that keep the server-side per-peer cap below the -// default. +// instead for clusters that keep the server-side per-peer cap below the default. +// The read timeout intentionally exceeds the proxy's script replay timeout so +// per-operation contexts, not socket deadlines, decide script cancellation. func DefaultElasticKVBackendOptions() BackendOptions { opts := DefaultBackendOptions() opts.PoolSize = defaultElasticKVPoolSize diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 68abbdc18..b0baf402f 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -1379,7 +1379,7 @@ func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() assert.Equal(t, 192, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) - assert.Equal(t, 35*time.Second, opts.ReadTimeout) + assert.Equal(t, defaultSecondaryScriptTimeout+blockingReadGrace, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout) } From 1978aad08464dd424c595dc30243189f9c69a922 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 13:52:39 +0900 Subject: [PATCH 10/18] redis-proxy: skip no-effect blocking replay --- docs/redis-proxy-deployment.md | 3 ++- proxy/blocking.go | 13 +++++++++- proxy/dualwrite.go | 19 +++++++++----- proxy/proxy_test.go | 45 +++++++++++++++++++++++++++++++--- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index b35b66d4c..da60c3761 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -38,7 +38,7 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-elastickv-pool-size` | `192` | ElasticKV backend connection pool size | | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` | | `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives one thirty-second of the shared write limit, capped at `3`, minimum `1` | -| `-secondary-blocking-replay-concurrency` | `0` | Mutating blocking-command replay sublimit. `0` uses remaining secondary backend pool capacity, capped at `32` | +| `-secondary-blocking-replay-concurrency` | `0` | Fallback mutating blocking-command replay sublimit. `0` uses remaining secondary backend pool capacity, capped at `32` | | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-secondary-script-queue-size` | `0` | Bounded queue for secondary Lua-script writes. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-secondary-blocking-replay-queue-size` | `0` | Bounded queue for mutating blocking-command replays. `0` derives `64 * concurrency`, clamped to `64..8192` | @@ -445,6 +445,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. - Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` at least `proxy replicas sharing one client IP * -elastickv-pool-size + 128`; PubSub and shadow PubSub use dedicated connections outside the command pool. With the HA compose defaults, use at least `512`. Keep `-secondary-write-concurrency` at or below the pool size. +- Successful `BZPOPMIN` / `BZPOPMAX` calls are replayed as `ZREM` on the normal write queue. Timeout/null blocking responses are not replayed because the primary made no mutation. - If script drops rise while backend pool waits stay at zero, the bottleneck is server-side Lua replay or wide-column cleanup, not connection acquisition. Keep `-secondary-script-concurrency` low; use `-secondary-script-timeout` for burst backlog and profile ElasticKV before raising concurrency. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. diff --git a/proxy/blocking.go b/proxy/blocking.go index be9d7e5d2..8698f783b 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -53,7 +53,10 @@ func parseBlockingMillisecondsArg(raw []byte) time.Duration { return time.Duration(millis) * time.Millisecond } -func shouldReplayBlockingToSecondary(cmd string) bool { +func shouldReplayBlockingToSecondary(cmd string, resp any) bool { + if !blockingResponseHasEffect(resp) { + return false + } return !strings.EqualFold(cmd, "XREAD") } @@ -77,3 +80,11 @@ func zsetPopKeyMember(resp any) (any, any, bool) { } return arr[0], arr[1], true } + +func blockingResponseHasEffect(resp any) bool { + if resp == nil { + return false + } + arr, ok := resp.([]any) + return !ok || len(arr) > 0 +} diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index 5f4972a00..a6a3fbc11 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -29,10 +29,10 @@ const ( // contention bounded; this is only tolerable in modes where the script write // is targeting the non-authoritative backend. maxScriptWriteGoroutines = 64 - // maxBlockingReplayGoroutines isolates mutating blocking command replays - // from normal secondary writes. Blocking replays may wait for the secondary - // to observe a producer write, and they hit the secondary's heavy-command - // limiter, so keep the default well below the normal write limit. + // maxBlockingReplayGoroutines isolates fallback mutating blocking-command + // replays from normal secondary writes. BZPOP success replays are translated + // to ordinary ZREM writes before they reach the secondary, so they use the + // normal write queue instead. maxBlockingReplayGoroutines = 32 // Async queues absorb short bursts without allowing an unavailable or slow // secondary to build an unbounded replay backlog. @@ -363,14 +363,14 @@ func (d *DualWriter) Blocking(ctx context.Context, cmd string, args [][]byte) (a if d.hasSecondaryWrite() { if replayCmd, replayArgs, ok := secondaryBlockingReplay(cmd, resp); ok { - d.goBlockingReplay(func(ctx context.Context) { + d.goTranslatedBlockingReplay(func(ctx context.Context) { d.writeSecondaryPositiveIntWithOptions(ctx, replayCmd, replayArgs, positiveIntReplayOptions{ initialDelay: blockingReplayInitialDelay, noEffectRetryWindow: blockingReplayNoEffectRetryWindow, deadlineAsMiss: true, }) }) - } else if shouldReplayBlockingToSecondary(cmd) { + } else if shouldReplayBlockingToSecondary(cmd, resp) { d.goBlockingReplay(func(ctx context.Context) { sCtx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() @@ -835,6 +835,13 @@ func (d *DualWriter) goBlockingReplay(fn func(context.Context)) { d.enqueueAsync(d.blockingReplayQueue, d.blockingReplayQueueSlots, asyncQueueBlocking, fn) } +func (d *DualWriter) goTranslatedBlockingReplay(fn func(context.Context)) { + if cap(d.blockingReplaySem) == 0 { + return + } + d.goWrite(fn) +} + // goShadow launches fn in a bounded shadow-read goroutine. func (d *DualWriter) goShadow(fn func()) { d.goShadowWithSem(fn) diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index b0baf402f..3c620f757 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -784,6 +784,36 @@ func TestDualWriter_Blocking_BZPopReplayShortTimeoutStillAttemptsZRem(t *testing metrics.CommandTotal.WithLabelValues("ZREM", "secondary", "miss")), 0.001) } +func TestDualWriter_Blocking_DoesNotReplayNoEffectBZPop(t *testing.T) { + primary := &timeoutCapturingBackend{ + name: "primary", + returnErr: redis.Nil, + } + secondary := newMockBackend("secondary") + + metrics := newTestMetrics() + d := NewDualWriter( + primary, + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: time.Second, + SecondaryBlockingReplayConcurrency: 1, + }, + metrics, + newTestSentry(), + testLogger, + ) + + resp, err := d.Blocking(context.Background(), "BZPOPMIN", [][]byte{[]byte("BZPOPMIN"), []byte("queue"), []byte("1")}) + assert.ErrorIs(t, err, redis.Nil) + assert.Nil(t, resp) + d.Close() + + assert.Equal(t, 0, secondary.CallCount()) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncQueueDepth.WithLabelValues(asyncQueueBlocking)), 0.001) +} + func TestNoEffectReplayRetryLimitIncludesJitterBudget(t *testing.T) { limit := noEffectReplayRetryLimit(context.Background(), blockingReplayNoEffectRetryWindow) assert.Positive(t, limit) @@ -799,7 +829,7 @@ func TestNoEffectReplayRetryLimitIncludesJitterBudget(t *testing.T) { assert.Greater(t, spent+retryBackoffWithMaxJitter(backoff), blockingReplayNoEffectRetryWindow) } -func TestDualWriter_BlockingReplayDoesNotConsumeWriteWorkers(t *testing.T) { +func TestDualWriter_BlockingTranslatedReplayUsesWriteQueue(t *testing.T) { primary := &timeoutCapturingBackend{ name: "primary", returnValue: []any{"queue", "job-1", "12.5"}, @@ -813,7 +843,7 @@ func TestDualWriter_BlockingReplayDoesNotConsumeWriteWorkers(t *testing.T) { secondary, ProxyConfig{ Mode: ModeDualWrite, - SecondaryTimeout: time.Second, + SecondaryTimeout: 5 * time.Second, SecondaryWriteConcurrency: 1, SecondaryBlockingReplayConcurrency: 1, SecondaryWriteQueueCapacity: 1, @@ -832,14 +862,21 @@ func TestDualWriter_BlockingReplayDoesNotConsumeWriteWorkers(t *testing.T) { }) <-started - _, err := d.Blocking(context.Background(), "BZPOPMIN", [][]byte{[]byte("BZPOPMIN"), []byte("queue"), []byte("5")}) + resp, err := d.Blocking(context.Background(), "BZPOPMIN", [][]byte{[]byte("BZPOPMIN"), []byte("queue"), []byte("5")}) assert.NoError(t, err) - assert.Eventually(t, func() bool { return secondary.CallCount() == 1 }, + assert.Equal(t, []any{"queue", "job-1", "12.5"}, resp) + assert.Eventually(t, func() bool { + return testutil.ToFloat64(metrics.AsyncQueueDepth.WithLabelValues(asyncQueueWrite)) == 1 + }, time.Second, 10*time.Millisecond) assert.InDelta(t, 1, testutil.ToFloat64(metrics.AsyncWorkersActive.WithLabelValues(asyncQueueWrite)), 0.001) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncWorkersActive.WithLabelValues(asyncQueueBlocking)), 0.001) + assert.Equal(t, 0, secondary.CallCount()) close(blocker) d.Close() + + assert.Equal(t, 1, secondary.CallCount()) } func TestDualWriter_Blocking_ReplaysXReadGroup(t *testing.T) { From 728c5b87a8d59e6d1eff154e5cad07cb9efd872b Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 14:14:26 +0900 Subject: [PATCH 11/18] redis-proxy: fast-path zpop replay cleanup --- adapter/redis.go | 2 ++ adapter/redis_command_specs.go | 5 +++++ adapter/redis_retry_test.go | 22 ++++++++++++++++++++++ adapter/redis_zset_cmds.go | 16 +++++++++++++++- docs/redis-proxy-deployment.md | 2 +- proxy/blocking.go | 13 +++++++++++-- proxy/dualwrite.go | 2 +- proxy/proxy_test.go | 17 +++++++++++------ 8 files changed, 68 insertions(+), 11 deletions(-) diff --git a/adapter/redis.go b/adapter/redis.go index e503d860d..8b84160a5 100644 --- a/adapter/redis.go +++ b/adapter/redis.go @@ -114,6 +114,8 @@ const ( minKeyedArgs = 2 ) +const cmdElasticKVZRemFast = "ELASTICKV.ZREMFAST" + const ( redisDispatchTimeout = 30 * time.Second // redisLuaDispatchTimeout gives EVAL/EVALSHA enough room for migration diff --git a/adapter/redis_command_specs.go b/adapter/redis_command_specs.go index f10176d63..b08459293 100644 --- a/adapter/redis_command_specs.go +++ b/adapter/redis_command_specs.go @@ -45,6 +45,10 @@ var redisCommandSpecs = []redisCommandSpec{ {Constant: cmdDBSize, Name: "dbsize", Arity: 1, Flags: []string{redisCmdFlagReadonly}, FirstKey: 0, LastKey: 0, Step: 0}, {Constant: cmdDel, Name: "del", Arity: -2, Flags: []string{redisCmdFlagWrite}, FirstKey: 1, LastKey: -1, Step: 1}, {Constant: cmdDiscard, Name: "discard", Arity: 1, Flags: []string{redisCmdFlagAdmin}, FirstKey: 0, LastKey: 0, Step: 0}, + // ELASTICKV.ZREMFAST is a secondary-replay-only ZREM variant for the + // Redis proxy. It skips the full wrong-type scan when the expected zset row + // is absent so stale BZPOP replays do not turn into wide collection probes. + {Constant: cmdElasticKVZRemFast, Name: "elastickv.zremfast", Arity: -3, Flags: []string{redisCmdFlagWrite}, FirstKey: 1, LastKey: 1, Step: 1}, {Constant: cmdEval, Name: "eval", Arity: -3, Flags: []string{redisCmdFlagWrite}, FirstKey: 0, LastKey: 0, Step: 0}, {Constant: cmdEvalSHA, Name: "evalsha", Arity: -3, Flags: []string{redisCmdFlagWrite}, FirstKey: 0, LastKey: 0, Step: 0}, {Constant: cmdExec, Name: "exec", Arity: 1, Flags: []string{redisCmdFlagAdmin}, FirstKey: 0, LastKey: 0, Step: 0}, @@ -254,6 +258,7 @@ func (r *RedisServer) buildRouteMap() map[string]func(redcon.Conn, redcon.Comman cmdZRevRangeByScore: r.zrevrangebyscore, cmdZScore: r.zscore, } + handlers[cmdElasticKVZRemFast] = r.elasticKVZRemFast m := make(map[string]func(redcon.Conn, redcon.Command), len(redisCommandSpecs)) for _, s := range redisCommandSpecs { h, ok := handlers[s.Constant] diff --git a/adapter/redis_retry_test.go b/adapter/redis_retry_test.go index dd5eab7f5..627f14146 100644 --- a/adapter/redis_retry_test.go +++ b/adapter/redis_retry_test.go @@ -749,3 +749,25 @@ func TestZRemDeletesWideColumnRows(t *testing.T) { require.True(t, exists) require.Equal(t, []redisZSetEntry{{Member: "b", Score: 2.0}}, zset.Entries) } + +func TestElasticKVZRemFastSkipsWrongTypeMiss(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + key := []byte("internal:zremfast:string") + require.NoError(t, st.PutAt(ctx, redisStrKey(key), encodeRedisStr([]byte("value"), nil), 1, 0)) + + coord := newRetryOnceCoordinator(st) + coord.clock.Observe(1) + srv := NewRedisServer(nil, "", st, coord, nil, nil) + + normalConn := &recordingConn{} + srv.zrem(normalConn, redcon.Command{Args: [][]byte{[]byte("ZREM"), key, []byte("member")}}) + require.Contains(t, normalConn.err, wrongTypeMessage) + + fastConn := &recordingConn{} + srv.elasticKVZRemFast(fastConn, redcon.Command{Args: [][]byte{[]byte(cmdElasticKVZRemFast), key, []byte("member")}}) + require.Empty(t, fastConn.err) + require.Equal(t, int64(0), fastConn.int) +} diff --git a/adapter/redis_zset_cmds.go b/adapter/redis_zset_cmds.go index 1c4aca8b4..76d70533c 100644 --- a/adapter/redis_zset_cmds.go +++ b/adapter/redis_zset_cmds.go @@ -1009,6 +1009,14 @@ func (r *RedisServer) zrangeRead(conn redcon.Conn, key []byte, start, stop int, } func (r *RedisServer) zrem(conn redcon.Conn, cmd redcon.Command) { + r.zremWithTypeProbe(conn, cmd, false) +} + +func (r *RedisServer) elasticKVZRemFast(conn redcon.Conn, cmd redcon.Command) { + r.zremWithTypeProbe(conn, cmd, true) +} + +func (r *RedisServer) zremWithTypeProbe(conn redcon.Conn, cmd redcon.Command, fastMiss bool) { if r.proxyToLeader(conn, cmd, cmd.Args[1]) { return } @@ -1017,7 +1025,13 @@ func (r *RedisServer) zrem(conn redcon.Conn, cmd redcon.Command) { var removed int if err := r.retryRedisWrite(ctx, func() error { readTS := r.readTS() - typ, err := r.keyTypeAtExpect(ctx, cmd.Args[1], readTS, redisTypeZSet) + var typ redisValueType + var err error + if fastMiss { + typ, err = r.keyTypeAtExpectFast(ctx, cmd.Args[1], readTS, redisTypeZSet) + } else { + typ, err = r.keyTypeAtExpect(ctx, cmd.Args[1], readTS, redisTypeZSet) + } if err != nil { return err } diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index da60c3761..891ff6313 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -445,7 +445,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. - Keep `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` at least `proxy replicas sharing one client IP * -elastickv-pool-size + 128`; PubSub and shadow PubSub use dedicated connections outside the command pool. With the HA compose defaults, use at least `512`. Keep `-secondary-write-concurrency` at or below the pool size. -- Successful `BZPOPMIN` / `BZPOPMAX` calls are replayed as `ZREM` on the normal write queue. Timeout/null blocking responses are not replayed because the primary made no mutation. +- Successful `BZPOPMIN` / `BZPOPMAX` calls are replayed to ElasticKV as an internal fast ZREM variant on the normal write queue. Regular client `ZREM` still keeps Redis wrong-type behavior; stale BZPOP replays avoid wide type scans. Timeout/null blocking responses are not replayed because the primary made no mutation. - If script drops rise while backend pool waits stay at zero, the bottleneck is server-side Lua replay or wide-column cleanup, not connection acquisition. Keep `-secondary-script-concurrency` low; use `-secondary-script-timeout` for burst backlog and profile ElasticKV before raising concurrency. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. diff --git a/proxy/blocking.go b/proxy/blocking.go index 8698f783b..204c3650e 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -10,6 +10,7 @@ import ( ) const blockingMultiPopMinArgs = 2 +const elasticKVZRemFastCommand = "ELASTICKV.ZREMFAST" type blockingTimeoutBackend interface { DoWithTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd @@ -60,19 +61,27 @@ func shouldReplayBlockingToSecondary(cmd string, resp any) bool { return !strings.EqualFold(cmd, "XREAD") } -func secondaryBlockingReplay(cmd string, resp any) (string, []any, bool) { +func secondaryBlockingReplay(cmd string, resp any, secondary Backend) (string, []any, bool) { switch strings.ToUpper(cmd) { case "BZPOPMIN", "BZPOPMAX": key, member, ok := zsetPopKeyMember(resp) if !ok { return "", nil, false } - return "ZREM", []any{"ZREM", key, member}, true + replayCmd := zremReplayCommandForSecondary(secondary) + return replayCmd, []any{replayCmd, key, member}, true default: return "", nil, false } } +func zremReplayCommandForSecondary(secondary Backend) string { + if secondary != nil && strings.EqualFold(secondary.Name(), "elastickv") { + return elasticKVZRemFastCommand + } + return "ZREM" +} + func zsetPopKeyMember(resp any) (any, any, bool) { arr, ok := resp.([]any) if !ok || len(arr) < 2 || arr[0] == nil || arr[1] == nil { diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index a6a3fbc11..c4ae650a2 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -362,7 +362,7 @@ func (d *DualWriter) Blocking(ctx context.Context, cmd string, args [][]byte) (a d.metrics.CommandTotal.WithLabelValues(cmd, d.primary.Name(), "ok").Inc() if d.hasSecondaryWrite() { - if replayCmd, replayArgs, ok := secondaryBlockingReplay(cmd, resp); ok { + if replayCmd, replayArgs, ok := secondaryBlockingReplay(cmd, resp, d.secondary); ok { d.goTranslatedBlockingReplay(func(ctx context.Context) { d.writeSecondaryPositiveIntWithOptions(ctx, replayCmd, replayArgs, positiveIntReplayOptions{ initialDelay: blockingReplayInitialDelay, diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 3c620f757..be9dc6674 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -640,17 +640,22 @@ func TestDualWriter_Blocking_UsesTimeoutAwareBackend(t *testing.T) { func TestDualWriter_Blocking_ReplaysBZPopAsZRem(t *testing.T) { for _, tc := range []struct { - cmd string + name string + cmd string + secondaryName string + expectedCmd string }{ - {cmd: "BZPOPMIN"}, - {cmd: "BZPOPMAX"}, + {name: "generic secondary BZPOPMIN", cmd: "BZPOPMIN", secondaryName: "secondary", expectedCmd: "ZREM"}, + {name: "generic secondary BZPOPMAX", cmd: "BZPOPMAX", secondaryName: "secondary", expectedCmd: "ZREM"}, + {name: "elastickv secondary BZPOPMIN", cmd: "BZPOPMIN", secondaryName: "elastickv", expectedCmd: elasticKVZRemFastCommand}, + {name: "elastickv secondary BZPOPMAX", cmd: "BZPOPMAX", secondaryName: "elastickv", expectedCmd: elasticKVZRemFastCommand}, } { - t.Run(tc.cmd, func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { primary := &timeoutCapturingBackend{ name: "primary", returnValue: []any{"queue", "job-1", "12.5"}, } - secondary := newMockBackend("secondary") + secondary := newMockBackend(tc.secondaryName) secondary.doFunc = makeCmd(int64(1), nil) metrics := newTestMetrics() @@ -676,7 +681,7 @@ func TestDualWriter_Blocking_ReplaysBZPopAsZRem(t *testing.T) { secondary.mu.Lock() got := append([]any(nil), secondary.calls[0]...) secondary.mu.Unlock() - assert.Equal(t, []any{"ZREM", "queue", "job-1"}, got) + assert.Equal(t, []any{tc.expectedCmd, "queue", "job-1"}, got) }) } } From 3fac31131061edf1537f59b3a5b7fdaa964150d1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 17:45:36 +0900 Subject: [PATCH 12/18] redis-proxy: isolate script replay deadlines --- adapter/redis_retry_test.go | 31 ++++ adapter/redis_zset_cmds.go | 30 ++- cmd/redis-proxy/main.go | 2 +- .../dashboards/elastickv-raft-status.json | 10 +- monitoring/hlc.go | 2 +- proxy/backend.go | 40 ++-- proxy/blocking.go | 4 + proxy/dualwrite.go | 86 ++++++++- proxy/leader_aware_backend.go | 12 ++ proxy/proxy.go | 2 +- proxy/proxy_test.go | 172 ++++++++++++++++-- proxy/pubsub.go | 2 +- 12 files changed, 341 insertions(+), 52 deletions(-) diff --git a/adapter/redis_retry_test.go b/adapter/redis_retry_test.go index 627f14146..7dff88b7e 100644 --- a/adapter/redis_retry_test.go +++ b/adapter/redis_retry_test.go @@ -771,3 +771,34 @@ func TestElasticKVZRemFastSkipsWrongTypeMiss(t *testing.T) { require.Empty(t, fastConn.err) require.Equal(t, int64(0), fastConn.int) } + +func TestElasticKVZRemFastRemovesLegacyBlobZSet(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + key := []byte("internal:zremfast:legacy") + payload, err := marshalZSetValue(redisZSetValue{ + Entries: []redisZSetEntry{ + {Member: "a", Score: 1}, + {Member: "b", Score: 2}, + }, + }) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, redisZSetKey(key), payload, 1, 0)) + + coord := newRetryOnceCoordinator(st) + coord.clock.Observe(1) + srv := NewRedisServer(nil, "", st, coord, nil, nil) + + conn := &recordingConn{} + srv.elasticKVZRemFast(conn, redcon.Command{Args: [][]byte{[]byte(cmdElasticKVZRemFast), key, []byte("a")}}) + + require.Empty(t, conn.err) + require.Equal(t, int64(1), conn.int) + + zset, exists, err := srv.loadZSetAt(ctx, key, snapshotTS(coord.clock, st)) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, []redisZSetEntry{{Member: "b", Score: 2}}, zset.Entries) +} diff --git a/adapter/redis_zset_cmds.go b/adapter/redis_zset_cmds.go index 76d70533c..cfc8be8a4 100644 --- a/adapter/redis_zset_cmds.go +++ b/adapter/redis_zset_cmds.go @@ -1025,13 +1025,7 @@ func (r *RedisServer) zremWithTypeProbe(conn redcon.Conn, cmd redcon.Command, fa var removed int if err := r.retryRedisWrite(ctx, func() error { readTS := r.readTS() - var typ redisValueType - var err error - if fastMiss { - typ, err = r.keyTypeAtExpectFast(ctx, cmd.Args[1], readTS, redisTypeZSet) - } else { - typ, err = r.keyTypeAtExpect(ctx, cmd.Args[1], readTS, redisTypeZSet) - } + typ, err := r.zremTypeAt(ctx, cmd.Args[1], readTS, fastMiss) if err != nil { return err } @@ -1060,6 +1054,28 @@ func (r *RedisServer) zremWithTypeProbe(conn redcon.Conn, cmd redcon.Command, fa conn.WriteInt(removed) } +func (r *RedisServer) zremTypeAt(ctx context.Context, key []byte, readTS uint64, fastMiss bool) (redisValueType, error) { + if !fastMiss { + return r.keyTypeAtExpect(ctx, key, readTS, redisTypeZSet) + } + typ, err := r.keyTypeAtExpectFast(ctx, key, readTS, redisTypeZSet) + if err != nil || typ != redisTypeNone { + return typ, err + } + return r.legacyZSetTypeAt(ctx, key, readTS) +} + +func (r *RedisServer) legacyZSetTypeAt(ctx context.Context, key []byte, readTS uint64) (redisValueType, error) { + exists, err := r.store.ExistsAt(ctx, redisZSetKey(key), readTS) + if err != nil { + return redisTypeNone, cockerrors.WithStack(err) + } + if !exists { + return redisTypeNone, nil + } + return r.applyTTLFilter(ctx, key, readTS, redisTypeZSet) +} + func (r *RedisServer) zremrangebyrank(conn redcon.Conn, cmd redcon.Command) { if r.proxyToLeader(conn, cmd, cmd.Args[1]) { return diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index 7c95c0bf1..d1b2df0c9 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -62,7 +62,7 @@ func run() error { flag.IntVar(&secondaryBlockingReplayQueueSize, "secondary-blocking-replay-queue-size", secondaryBlockingReplayQueueSize, "Maximum queued asynchronous secondary mutating blocking-command replays (0 = derived from blocking replay concurrency)") flag.StringVar(&modeStr, "mode", "dual-write", "Proxy mode: redis-only, dual-write, dual-write-shadow, elastickv-primary, elastickv-only") flag.DurationVar(&cfg.SecondaryTimeout, "secondary-timeout", cfg.SecondaryTimeout, "Secondary write timeout") - flag.DurationVar(&cfg.SecondaryScriptTimeout, "secondary-script-timeout", cfg.SecondaryScriptTimeout, "Secondary Lua-script write timeout (0 = secondary-timeout)") + flag.DurationVar(&cfg.SecondaryScriptTimeout, "secondary-script-timeout", cfg.SecondaryScriptTimeout, "Secondary Lua-script write timeout for Redis or ElasticKV replay, including transaction scripts (0 = secondary-timeout)") flag.DurationVar(&cfg.ShadowTimeout, "shadow-timeout", cfg.ShadowTimeout, "Shadow read timeout") flag.StringVar(&cfg.SentryDSN, "sentry-dsn", cfg.SentryDSN, "Sentry DSN (empty = disabled)") flag.StringVar(&cfg.SentryEnv, "sentry-env", cfg.SentryEnv, "Sentry environment") diff --git a/monitoring/grafana/dashboards/elastickv-raft-status.json b/monitoring/grafana/dashboards/elastickv-raft-status.json index e9c1f065f..677e25e39 100644 --- a/monitoring/grafana/dashboards/elastickv-raft-status.json +++ b/monitoring/grafana/dashboards/elastickv-raft-status.json @@ -1071,7 +1071,7 @@ }, { "datasource": "$datasource", - "description": "Worst wall_now - physicalCeiling across nodes. Negative is healthy (ceiling sits in the future). Alert on > -0.5s for HLC-4 (i) approaching violation, > 0s for the (iii) fence having tripped on at least one node.", + "description": "Worst wall_now - physicalCeiling across nodes after the first HLC lease has landed. Negative is healthy (ceiling sits in the future). Alert on > -5s with physicalCeiling > 0 for renewal stalls, > 0s for the (iii) fence having tripped on at least one node.", "fieldConfig": { "defaults": { "color": { @@ -1086,7 +1086,7 @@ }, { "color": "yellow", - "value": -0.5 + "value": -5 }, { "color": "red", @@ -1120,7 +1120,7 @@ { "datasource": "$datasource", "editorMode": "code", - "expr": "max(elastickv_hlc_wall_skew_seconds{job=\"elastickv\",node_id=~\"$node_id\"})", + "expr": "max(elastickv_hlc_wall_skew_seconds{job=\"elastickv\",node_id=~\"$node_id\"} and elastickv_hlc_physical_ceiling_seconds{job=\"elastickv\",node_id=~\"$node_id\"} > 0)", "refId": "A", "instant": true } @@ -1240,7 +1240,7 @@ }, { "datasource": "$datasource", - "description": "wall_now minus physicalCeiling per node. Negative = healthy (ceiling sits in the future). Zero or positive = the (iii) fence rejection boundary has been reached. Pre-bootstrap (ceiling=0) is held at 0 by the observer to avoid a spurious ~55-year skew on cold start.", + "description": "wall_now minus physicalCeiling per node. Negative = healthy after physicalCeiling > 0. Zero or positive = the (iii) fence rejection boundary has been reached. Pre-bootstrap (ceiling=0) is held at 0 by the observer to avoid a spurious ~55-year skew on cold start; alerts should also require physicalCeiling > 0.", "fieldConfig": { "defaults": { "color": { @@ -1264,7 +1264,7 @@ }, { "color": "yellow", - "value": -0.5 + "value": -5 }, { "color": "red", diff --git a/monitoring/hlc.go b/monitoring/hlc.go index 08c017602..c75cf1f52 100644 --- a/monitoring/hlc.go +++ b/monitoring/hlc.go @@ -68,7 +68,7 @@ func newHLCMetrics(registerer prometheus.Registerer) *HLCMetrics { }), wallSkewSeconds: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "elastickv_hlc_wall_skew_seconds", - Help: "wall_now - physicalCeiling, in seconds. Negative is healthy (ceiling is in the future); zero or positive means NextFenced is at-or-past the rejection boundary. Alert on > -5 to catch renewal stalls before the HLC-4 (iii) fence trips.", + Help: "wall_now - physicalCeiling, in seconds. Negative is healthy after physicalCeiling > 0; pre-bootstrap physicalCeiling == 0 is held at 0. Alert on wallSkewSeconds > -5 and physicalCeilingSeconds > 0 to catch renewal stalls before the HLC-4 (iii) fence trips.", }), nextFencedRejectionsTotal: prometheus.NewCounter(prometheus.CounterOpts{ Name: "elastickv_hlc_next_fenced_rejections_total", diff --git a/proxy/backend.go b/proxy/backend.go index c47b935a2..37b9a47a1 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -10,14 +10,13 @@ import ( ) const ( - defaultPoolSize = 128 - defaultElasticKVPoolSize = 192 - defaultDialTimeout = 5 * time.Second - defaultReadTimeout = 3 * time.Second - blockingReadGrace = 10 * time.Second - defaultElasticKVReadTimeout = defaultSecondaryScriptTimeout + blockingReadGrace - defaultWriteTimeout = 3 * time.Second - respProtocolV2 = 2 + defaultPoolSize = 128 + defaultElasticKVPoolSize = 192 + defaultDialTimeout = 5 * time.Second + defaultReadTimeout = 3 * time.Second + blockingReadGrace = 10 * time.Second + defaultWriteTimeout = 3 * time.Second + respProtocolV2 = 2 ) // Backend abstracts a Redis-protocol endpoint (real Redis or ElasticKV). @@ -73,17 +72,14 @@ func DefaultBackendOptions() BackendOptions { } // DefaultElasticKVBackendOptions returns defaults for proxy backends that -// connect to ElasticKV's Redis adapter. Production dual-write deployments -// should run the cluster with ELASTICKV_REDIS_PER_PEER_CONNECTIONS sized for -// every proxy replica that may share one client IP, plus dedicated PubSub and -// shadow PubSub connections outside this command pool. Lower the proxy pool -// instead for clusters that keep the server-side per-peer cap below the default. -// The read timeout intentionally exceeds the proxy's script replay timeout so -// per-operation contexts, not socket deadlines, decide script cancellation. +// connect to ElasticKV's Redis adapter. Production dual-write deployments should +// run the cluster with ELASTICKV_REDIS_PER_PEER_CONNECTIONS sized for every +// proxy replica that may share one client IP, plus dedicated PubSub and shadow +// PubSub connections outside this command pool. Lower the proxy pool instead +// for clusters that keep the server-side per-peer cap below the default. func DefaultElasticKVBackendOptions() BackendOptions { opts := DefaultBackendOptions() opts.PoolSize = defaultElasticKVPoolSize - opts.ReadTimeout = defaultElasticKVReadTimeout return opts } @@ -140,7 +136,17 @@ func effectiveBlockingReadTimeout(timeout time.Duration) time.Duration { } func (b *RedisBackend) Pipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { - pipe := b.client.Pipeline() + return b.pipeline(ctx, b.client, cmds) +} + +// PipelineWithTimeout executes a pipeline using a per-call socket timeout +// override for async replay classes whose deadline exceeds the backend default. +func (b *RedisBackend) PipelineWithTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { + return b.pipeline(ctx, b.client.WithTimeout(effectiveBlockingReadTimeout(timeout)), cmds) +} + +func (b *RedisBackend) pipeline(ctx context.Context, client *redis.Client, cmds [][]any) ([]*redis.Cmd, error) { + pipe := client.Pipeline() results := make([]*redis.Cmd, len(cmds)) for i, args := range cmds { results[i] = pipe.Do(ctx, args...) diff --git a/proxy/blocking.go b/proxy/blocking.go index 204c3650e..4c24f2b6b 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -16,6 +16,10 @@ type blockingTimeoutBackend interface { DoWithTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd } +type pipelineTimeoutBackend interface { + PipelineWithTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) +} + func blockingCommandTimeout(cmd string, args [][]byte) time.Duration { switch strings.ToUpper(cmd) { case "BLPOP", "BRPOP", "BRPOPLPUSH", "BLMOVE", "BZPOPMIN", "BZPOPMAX": diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index c4ae650a2..ca549991b 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -444,7 +444,7 @@ func (d *DualWriter) writeSecondary(sCtx context.Context, cmd string, iArgs []an var usedNOSCRIPTFallback bool args := iArgs for ; ; attempt++ { - result := d.secondary.Do(sCtx, args...) + result := d.secondaryDo(sCtx, cmd, args) _, sErr = result.Result() if isNoScriptError(sErr) { // After a successful NOSCRIPT→EVAL resolution, retries reuse the @@ -453,7 +453,7 @@ func (d *DualWriter) writeSecondary(sCtx context.Context, cmd string, iArgs []an if fallbackArgs, ok := d.evalFallbackArgs(cmd, args); ok { usedNOSCRIPTFallback = true args = fallbackArgs - result = d.secondary.Do(sCtx, args...) + result = d.secondaryDo(sCtx, cmd, args) _, sErr = result.Result() } } @@ -502,7 +502,7 @@ func (d *DualWriter) writeSecondaryPositiveIntWithOptions(sCtx context.Context, var attempt int for ; ; attempt++ { var resp any - result := d.secondary.Do(sCtx, iArgs...) + result := d.secondaryDo(sCtx, cmd, iArgs) resp, sErr = result.Result() if ok, err := positiveIntReplayResult(resp, sErr); ok { elapsed := time.Since(start) @@ -671,7 +671,7 @@ func (d *DualWriter) writeSecondaryPipeline(sCtx context.Context, cmds [][]any) var sErr error var attempt int for ; ; attempt++ { - results, pErr := d.secondary.Pipeline(sCtx, cmds) + results, pErr := d.secondaryPipeline(sCtx, cmds) sErr = secondaryPipelineError(results, pErr) retryReason, retryLimit := secondaryPipelineRetryReasonAndLimit(sErr) if retryReason == "" { @@ -836,9 +836,6 @@ func (d *DualWriter) goBlockingReplay(fn func(context.Context)) { } func (d *DualWriter) goTranslatedBlockingReplay(fn func(context.Context)) { - if cap(d.blockingReplaySem) == 0 { - return - } d.goWrite(fn) } @@ -847,11 +844,19 @@ func (d *DualWriter) goShadow(fn func()) { d.goShadowWithSem(fn) } -// goAsync queues fn with the write class (for txn replay). +// goAsync queues fn with the write class (for txn replay without scripts). func (d *DualWriter) goAsync(fn func(context.Context)) { d.goWrite(fn) } +func (d *DualWriter) goTxnReplay(cmds [][]any, fn func(context.Context)) { + if pipelineContainsScript(cmds) { + d.goScript(fn) + return + } + d.goAsync(fn) +} + func secondaryScriptTimeout(cfg ProxyConfig) time.Duration { if cfg.SecondaryScriptTimeout > 0 { return cfg.SecondaryScriptTimeout @@ -859,6 +864,71 @@ func secondaryScriptTimeout(cfg ProxyConfig) time.Duration { return cfg.SecondaryTimeout } +func (d *DualWriter) secondaryDo(ctx context.Context, cmd string, args []any) *redis.Cmd { + timeout := d.secondaryCommandReadTimeout(cmd) + if timeout > 0 { + if backend, ok := d.secondary.(blockingTimeoutBackend); ok { + return backend.DoWithTimeout(ctx, timeout, args...) + } + } + return d.secondary.Do(ctx, args...) +} + +func (d *DualWriter) secondaryPipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { + timeout := d.secondaryPipelineReadTimeout(cmds) + if timeout > 0 { + if backend, ok := d.secondary.(pipelineTimeoutBackend); ok { + results, err := backend.PipelineWithTimeout(ctx, timeout, cmds) + if err != nil { + return results, fmt.Errorf("secondary pipeline with timeout: %w", err) + } + return results, nil + } + } + results, err := d.secondary.Pipeline(ctx, cmds) + if err != nil { + return results, fmt.Errorf("secondary pipeline: %w", err) + } + return results, nil +} + +func (d *DualWriter) secondaryCommandReadTimeout(cmd string) time.Duration { + if isRedisScriptCommandName(cmd) { + return secondaryScriptTimeout(d.cfg) + } + return d.cfg.SecondaryTimeout +} + +func (d *DualWriter) secondaryPipelineReadTimeout(cmds [][]any) time.Duration { + if pipelineContainsScript(cmds) { + return secondaryScriptTimeout(d.cfg) + } + return d.cfg.SecondaryTimeout +} + +func pipelineContainsScript(cmds [][]any) bool { + for _, args := range cmds { + if len(args) == 0 { + continue + } + if isRedisScriptCommandName(redisCommandName(args[0])) { + return true + } + } + return false +} + +func redisCommandName(arg any) string { + switch v := arg.(type) { + case []byte: + return string(v) + case string: + return v + default: + return fmt.Sprint(v) + } +} + func (d *DualWriter) startBackendPoolSampler() { backends := backendPoolSampleBackends(d.primary, d.secondary) if len(backends) == 0 { diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index 05bd6e6b0..f15a0f7d0 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -437,10 +437,22 @@ func (b *LeaderAwareRedisBackend) doWithTimeoutOnce(ctx context.Context, timeout // Pipeline forwards a batch to the current leader. func (b *LeaderAwareRedisBackend) Pipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { + return b.pipeline(ctx, 0, cmds) +} + +// PipelineWithTimeout forwards a batch with a per-call socket timeout override. +func (b *LeaderAwareRedisBackend) PipelineWithTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { + return b.pipeline(ctx, effectiveBlockingReadTimeout(timeout), cmds) +} + +func (b *LeaderAwareRedisBackend) pipeline(ctx context.Context, readTimeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { cli := b.currentClient() if cli == nil { return nil, ErrNoLeaderBackend } + if readTimeout > 0 { + cli = cli.WithTimeout(readTimeout) + } pipe := cli.Pipeline() results := make([]*redis.Cmd, len(cmds)) for i, args := range cmds { diff --git a/proxy/proxy.go b/proxy/proxy.go index b586ff090..47bcf0176 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -432,7 +432,7 @@ func (p *ProxyServer) execTxn(conn redcon.Conn, state *proxyConnState) { // Async replay to secondary (bounded) if p.dual.hasSecondaryWrite() { - p.dual.goAsync(func(ctx context.Context) { + p.dual.goTxnReplay(cmds, func(ctx context.Context) { p.dual.writeSecondaryPipeline(ctx, cmds) }) } diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index be9dc6674..6934e8629 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -531,13 +531,17 @@ func TestDualWriter_Read_NoShadowInDualWrite(t *testing.T) { } type timeoutCapturingBackend struct { - name string - timeout time.Duration - args []any - doCalls int - doWithCalls int - returnValue any - returnErr error + name string + timeout time.Duration + pipelineTimeout time.Duration + args []any + pipelineCmds [][]any + doCalls int + doWithCalls int + pipelineCalls int + pipelineWithCalls int + returnValue any + returnErr error } func (b *timeoutCapturingBackend) Do(ctx context.Context, args ...any) *redis.Cmd { @@ -566,6 +570,8 @@ func (b *timeoutCapturingBackend) DoWithTimeout(ctx context.Context, timeout tim } func (b *timeoutCapturingBackend) Pipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { + b.pipelineCalls++ + b.pipelineCmds = clonePipelineArgs(cmds) results := make([]*redis.Cmd, len(cmds)) for i, args := range cmds { results[i] = b.Do(ctx, args...) @@ -573,9 +579,34 @@ func (b *timeoutCapturingBackend) Pipeline(ctx context.Context, cmds [][]any) ([ return results, nil } +func (b *timeoutCapturingBackend) PipelineWithTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { + b.pipelineWithCalls++ + b.pipelineTimeout = timeout + b.pipelineCmds = clonePipelineArgs(cmds) + results := make([]*redis.Cmd, len(cmds)) + for i, args := range cmds { + cmd := redis.NewCmd(ctx, args...) + if b.returnErr != nil && i == len(cmds)-1 { + cmd.SetErr(b.returnErr) + } else { + cmd.SetVal(b.returnValue) + } + results[i] = cmd + } + return results, nil +} + func (b *timeoutCapturingBackend) Close() error { return nil } func (b *timeoutCapturingBackend) Name() string { return b.name } +func clonePipelineArgs(cmds [][]any) [][]any { + out := make([][]any, len(cmds)) + for i, args := range cmds { + out[i] = append([]any(nil), args...) + } + return out +} + func TestBlockingCommandTimeout(t *testing.T) { tests := []struct { name string @@ -951,7 +982,7 @@ func TestDualWriter_Blocking_DoesNotReplayXRead(t *testing.T) { func TestDualWriter_Blocking_DoesNotReplayWhenBlockingReplayDisabled(t *testing.T) { primary := &timeoutCapturingBackend{ name: "primary", - returnValue: []any{"queue", "job-1", "12.5"}, + returnValue: []any{"stream-result"}, } secondary := newMockBackend("secondary") @@ -965,15 +996,50 @@ func TestDualWriter_Blocking_DoesNotReplayWhenBlockingReplayDisabled(t *testing. testLogger, ) - resp, err := d.Blocking(context.Background(), "BZPOPMIN", [][]byte{[]byte("BZPOPMIN"), []byte("queue"), []byte("5")}) + args := [][]byte{ + []byte("XREADGROUP"), []byte("GROUP"), []byte("g"), []byte("c"), + []byte("BLOCK"), []byte("1000"), []byte("STREAMS"), []byte("jobs"), []byte(">"), + } + resp, err := d.Blocking(context.Background(), "XREADGROUP", args) assert.NoError(t, err) - assert.Equal(t, []any{"queue", "job-1", "12.5"}, resp) + assert.Equal(t, []any{"stream-result"}, resp) d.Close() assert.Equal(t, 0, secondary.CallCount()) assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) } +func TestDualWriter_BlockingTranslatedReplayUsesWriteQueueWhenBlockingReplayDisabled(t *testing.T) { + primary := &timeoutCapturingBackend{ + name: "primary", + returnValue: []any{"queue", "job-1", "12.5"}, + } + secondary := newMockBackend("secondary") + secondary.doFunc = makeCmd(int64(1), nil) + + metrics := newTestMetrics() + d := NewDualWriter( + primary, + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: time.Second, + SecondaryWriteConcurrency: 1, + }, + metrics, + newTestSentry(), + testLogger, + ) + + resp, err := d.Blocking(context.Background(), "BZPOPMIN", [][]byte{[]byte("BZPOPMIN"), []byte("queue"), []byte("5")}) + assert.NoError(t, err) + assert.Equal(t, []any{"queue", "job-1", "12.5"}, resp) + d.Close() + + assert.Equal(t, 1, secondary.CallCount()) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDropsByQueue.WithLabelValues(asyncQueueBlocking, asyncDropQueueFull)), 0.001) +} + func TestDualWriter_GoAsync_QueuesBurstBeforeDropping(t *testing.T) { primary := newMockBackend("primary") primary.doFunc = makeCmd("OK", nil) @@ -1232,6 +1298,90 @@ func TestDualWriter_ScriptQueueUsesScriptTimeout(t *testing.T) { metrics.AsyncDropsByQueue.WithLabelValues(asyncQueueScript, asyncDropExpired)), 0.001) } +func TestDualWriter_SecondaryUsesPerCommandSocketTimeouts(t *testing.T) { + secondary := &timeoutCapturingBackend{name: "secondary", returnValue: "OK"} + d := NewDualWriter( + newMockBackend("primary"), + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: 5 * time.Minute, + }, + newTestMetrics(), + newTestSentry(), + testLogger, + ) + + d.writeSecondary(context.Background(), "SET", []any{[]byte("SET"), []byte("k"), []byte("v")}) + assert.Equal(t, 1, secondary.doWithCalls) + assert.Equal(t, 30*time.Second, secondary.timeout) + + d.writeSecondary(context.Background(), "EVALSHA", []any{[]byte("EVALSHA"), []byte("deadbeef"), []byte("0")}) + assert.Equal(t, 2, secondary.doWithCalls) + assert.Equal(t, 5*time.Minute, secondary.timeout) +} + +func TestDualWriter_TxnReplayWithScriptUsesScriptQueueAndTimeout(t *testing.T) { + secondary := &timeoutCapturingBackend{name: "secondary", returnValue: "OK"} + metrics := newTestMetrics() + d := NewDualWriter( + newMockBackend("primary"), + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: 5 * time.Minute, + SecondaryWriteConcurrency: 1, + SecondaryScriptConcurrency: 1, + SecondaryWriteQueueCapacity: 1, + SecondaryScriptQueueCapacity: 1, + }, + metrics, + newTestSentry(), + testLogger, + ) + + cmds := [][]any{{"MULTI"}, {"EVALSHA", "deadbeef", "0"}, {"EXEC"}} + d.goTxnReplay(cmds, func(ctx context.Context) { + d.writeSecondaryPipeline(ctx, cmds) + }) + d.Close() + + assert.Equal(t, 1, secondary.pipelineWithCalls) + assert.Equal(t, 5*time.Minute, secondary.pipelineTimeout) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDropsByQueue.WithLabelValues(asyncQueueScript, asyncDropQueueFull)), 0.001) +} + +func TestDualWriter_TxnReplayWithoutScriptUsesWriteQueueAndTimeout(t *testing.T) { + secondary := &timeoutCapturingBackend{name: "secondary", returnValue: "OK"} + d := NewDualWriter( + newMockBackend("primary"), + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: 5 * time.Minute, + SecondaryWriteConcurrency: 1, + SecondaryScriptConcurrency: 1, + SecondaryWriteQueueCapacity: 1, + SecondaryScriptQueueCapacity: 1, + }, + newTestMetrics(), + newTestSentry(), + testLogger, + ) + + cmds := [][]any{{"MULTI"}, {"SET", "k", "v"}, {"EXEC"}} + d.goTxnReplay(cmds, func(ctx context.Context) { + d.writeSecondaryPipeline(ctx, cmds) + }) + d.Close() + + assert.Equal(t, 1, secondary.pipelineWithCalls) + assert.Equal(t, 30*time.Second, secondary.pipelineTimeout) +} + func TestDualWriter_ExpiredQueuedWorkDoesNotWaitForSemaphore(t *testing.T) { metrics := newTestMetrics() d := &DualWriter{ @@ -1421,7 +1571,7 @@ func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() assert.Equal(t, 192, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) - assert.Equal(t, defaultSecondaryScriptTimeout+blockingReadGrace, opts.ReadTimeout) + assert.Equal(t, 3*time.Second, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout) } diff --git a/proxy/pubsub.go b/proxy/pubsub.go index 54ab079f0..59245246d 100644 --- a/proxy/pubsub.go +++ b/proxy/pubsub.go @@ -470,7 +470,7 @@ func (s *pubsubSession) execTxn() { s.writeMu.Unlock() if s.proxy.dual.hasSecondaryWrite() { - s.proxy.dual.goAsync(func(ctx context.Context) { + s.proxy.dual.goTxnReplay(cmds, func(ctx context.Context) { s.proxy.dual.writeSecondaryPipeline(ctx, cmds) }) } From b11f4a189571cae80846669072ab0e3eb6fad035 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 19:41:26 +0900 Subject: [PATCH 13/18] proxy: bound secondary replay timeouts --- adapter/redis_workpool.go | 2 +- adapter/redis_workpool_test.go | 2 +- cmd/redis-proxy/main.go | 20 ++++++++ cmd/redis-proxy/main_test.go | 33 ++++++++++++ docs/redis-proxy-deployment.md | 4 +- proxy/backend.go | 8 +++ proxy/blocking.go | 15 ++++-- proxy/command.go | 2 +- proxy/dualwrite.go | 67 +++++++++++++++++++----- proxy/leader_aware_backend.go | 25 +++++++++ proxy/proxy_test.go | 93 +++++++++++++++++++++++++++++++--- 11 files changed, 244 insertions(+), 27 deletions(-) diff --git a/adapter/redis_workpool.go b/adapter/redis_workpool.go index b28f32b33..a60614a87 100644 --- a/adapter/redis_workpool.go +++ b/adapter/redis_workpool.go @@ -66,7 +66,7 @@ func isRedisHeavyCommand(name string) bool { cmdXRead, cmdXRange, cmdXRevRange, cmdZCount, cmdZPopMin, cmdZRange, cmdZRangeByScore, cmdZRevRange, cmdZRevRangeByScore, - cmdZRem, cmdZRemRangeByRank, cmdZRemRangeByScore, + cmdZRem, cmdElasticKVZRemFast, cmdZRemRangeByRank, cmdZRemRangeByScore, cmdBZPopMin: return true default: diff --git a/adapter/redis_workpool_test.go b/adapter/redis_workpool_test.go index d9a208da9..77c94dd9a 100644 --- a/adapter/redis_workpool_test.go +++ b/adapter/redis_workpool_test.go @@ -91,7 +91,7 @@ func TestRedisHeavyCommandClassification(t *testing.T) { cmdEval, cmdEvalSHA, cmdKeys, cmdScan, cmdHGetAll, cmdLRange, cmdSMembers, cmdXRead, cmdXRange, cmdXRevRange, cmdZRange, cmdZCount, cmdZPopMin, cmdZRangeByScore, cmdZRevRange, cmdZRevRangeByScore, - cmdZRem, cmdZRemRangeByRank, cmdZRemRangeByScore, cmdBZPopMin, + cmdZRem, cmdElasticKVZRemFast, cmdZRemRangeByRank, cmdZRemRangeByScore, cmdBZPopMin, cmdDBSize, } { require.True(t, isRedisHeavyCommand(cmd), cmd) diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index d1b2df0c9..0db83f5a2 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -24,6 +24,7 @@ const ( secondaryWriteConcurrencyDivisor = 2 secondaryScriptConcurrencyDivisor = 32 secondaryScriptConcurrencyCap = 3 + maxSecondaryScriptTimeout = 5 * time.Minute ) func main() { @@ -71,6 +72,7 @@ func run() error { flag.Parse() mode, resolvedWriteConcurrency, resolvedScriptConcurrency, resolvedBlockingReplayConcurrency, err := resolveRuntimeOptions( + cfg, modeStr, primaryPoolSize, elasticKVPoolSize, @@ -170,11 +172,15 @@ func run() error { } func resolveRuntimeOptions( + cfg proxy.ProxyConfig, modeStr string, primaryPoolSize, elasticKVPoolSize int, secondaryWriteConcurrency, secondaryScriptConcurrency, secondaryBlockingReplayConcurrency int, secondaryWriteQueueSize, secondaryScriptQueueSize, secondaryBlockingReplayQueueSize int, ) (proxy.ProxyMode, int, int, int, error) { + if err := validateRuntimeTimeouts(cfg); err != nil { + return 0, 0, 0, 0, err + } mode, err := parseRuntimeOptions( modeStr, primaryPoolSize, @@ -210,6 +216,20 @@ func resolveRuntimeOptions( return mode, writeConcurrency, scriptConcurrency, blockingReplayConcurrency, nil } +func validateRuntimeTimeouts(cfg proxy.ProxyConfig) error { + if cfg.SecondaryScriptTimeout < 0 { + return fmt.Errorf("secondary-script-timeout must be non-negative: %s", cfg.SecondaryScriptTimeout) + } + effectiveScriptTimeout := cfg.SecondaryScriptTimeout + if effectiveScriptTimeout == 0 { + effectiveScriptTimeout = cfg.SecondaryTimeout + } + if effectiveScriptTimeout > maxSecondaryScriptTimeout { + return fmt.Errorf("secondary-script-timeout %s exceeds ElasticKV Lua dispatch cap %s", effectiveScriptTimeout, maxSecondaryScriptTimeout) + } + return nil +} + func parseRuntimeOptions( modeStr string, primaryPoolSize, elasticKVPoolSize int, diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 76e7c8118..57c7d8f84 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -2,6 +2,7 @@ package main import ( "testing" + "time" "github.com/bootjp/elastickv/proxy" "github.com/stretchr/testify/assert" @@ -36,6 +37,38 @@ func TestParseRuntimeOptionsRejectsNegativeSecondaryQueueSize(t *testing.T) { assert.Contains(t, err.Error(), "secondary-blocking-replay-queue-size") } +func TestValidateRuntimeTimeoutsRejectsUnsupportedScriptTimeout(t *testing.T) { + require.NoError(t, validateRuntimeTimeouts(proxy.ProxyConfig{ + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: 0, + })) + require.NoError(t, validateRuntimeTimeouts(proxy.ProxyConfig{ + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: maxSecondaryScriptTimeout, + })) + + err := validateRuntimeTimeouts(proxy.ProxyConfig{ + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: -time.Second, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "secondary-script-timeout") + + err = validateRuntimeTimeouts(proxy.ProxyConfig{ + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: maxSecondaryScriptTimeout + time.Nanosecond, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "ElasticKV Lua dispatch cap") + + err = validateRuntimeTimeouts(proxy.ProxyConfig{ + SecondaryTimeout: maxSecondaryScriptTimeout + time.Nanosecond, + SecondaryScriptTimeout: 0, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "ElasticKV Lua dispatch cap") +} + func TestValidateSecondaryConcurrency(t *testing.T) { require.NoError(t, validateSecondaryConcurrency(proxy.ModeDualWrite, 128, 8, 4, 2, 4)) require.NoError(t, validateSecondaryConcurrency(proxy.ModeDualWrite, 128, 1, 1, 1, 0)) diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index 891ff6313..cae6b2dc1 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -44,7 +44,7 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-secondary-blocking-replay-queue-size` | `0` | Bounded queue for mutating blocking-command replays. `0` derives `64 * concurrency`, clamped to `64..8192` | | `-mode` | `dual-write` | Proxy mode (see below) | | `-secondary-timeout` | `30s` | End-to-end secondary write deadline, including queue wait | -| `-secondary-script-timeout` | `5m` | End-to-end secondary Lua-script write deadline, including queue wait. `0` follows `-secondary-timeout` | +| `-secondary-script-timeout` | `5m` | End-to-end secondary Lua-script write deadline, including queue wait. `0` follows `-secondary-timeout`; the effective value must not exceed the ElasticKV Lua dispatch cap (`5m`) | | `-shadow-timeout` | `3s` | Shadow read timeout | | `-sentry-dsn` | (empty) | Sentry DSN (empty = disabled) | | `-sentry-env` | (empty) | Sentry environment name | @@ -421,7 +421,7 @@ groups: | ElasticKV connection pool size | 192 | Default per-leader command pool; leave server per-peer headroom for dedicated PubSub connections | | ElasticKV Redis per-peer connection cap | 512 | Default server-side cap: two proxy replicas at pool `192`, plus dedicated PubSub/shadow PubSub headroom | | Dial timeout | 5s | Backend connection timeout | -| Read timeout | Redis: 3s, ElasticKV: 5m10s | Backend read timeout; ElasticKV keeps 10s grace beyond `-secondary-script-timeout` | +| Read timeout | Redis: 3s, ElasticKV: 3s | Default backend read timeout. Async secondary replays override the per-call read timeout with the remaining queue deadline; blocking commands add a 10s read grace to their requested wait | | Write timeout | 3s | Backend write timeout | | Async write concurrency fallback | 4096 | Package fallback; the command derives a lower limit from backend pool size | | Shadow read goroutine limit | 1024 | Max concurrent shadow comparisons | diff --git a/proxy/backend.go b/proxy/backend.go index 37b9a47a1..d4818cd18 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -128,6 +128,10 @@ func (b *RedisBackend) DoWithTimeout(ctx context.Context, timeout time.Duration, return b.client.WithTimeout(effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) } +func (b *RedisBackend) DoWithReadTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { + return b.client.WithTimeout(timeout).Do(ctx, args...) +} + func effectiveBlockingReadTimeout(timeout time.Duration) time.Duration { if timeout == 0 { return 0 @@ -145,6 +149,10 @@ func (b *RedisBackend) PipelineWithTimeout(ctx context.Context, timeout time.Dur return b.pipeline(ctx, b.client.WithTimeout(effectiveBlockingReadTimeout(timeout)), cmds) } +func (b *RedisBackend) PipelineWithReadTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { + return b.pipeline(ctx, b.client.WithTimeout(timeout), cmds) +} + func (b *RedisBackend) pipeline(ctx context.Context, client *redis.Client, cmds [][]any) ([]*redis.Cmd, error) { pipe := client.Pipeline() results := make([]*redis.Cmd, len(cmds)) diff --git a/proxy/blocking.go b/proxy/blocking.go index 4c24f2b6b..252e6d8e0 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -10,14 +10,21 @@ import ( ) const blockingMultiPopMinArgs = 2 -const elasticKVZRemFastCommand = "ELASTICKV.ZREMFAST" +const ( + elasticKVZRemFastCommand = "ELASTICKV.ZREMFAST" + zremReplayCommand = "ZREM" +) type blockingTimeoutBackend interface { DoWithTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd } -type pipelineTimeoutBackend interface { - PipelineWithTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) +type readTimeoutBackend interface { + DoWithReadTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd +} + +type pipelineReadTimeoutBackend interface { + PipelineWithReadTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) } func blockingCommandTimeout(cmd string, args [][]byte) time.Duration { @@ -83,7 +90,7 @@ func zremReplayCommandForSecondary(secondary Backend) string { if secondary != nil && strings.EqualFold(secondary.Name(), "elastickv") { return elasticKVZRemFastCommand } - return "ZREM" + return zremReplayCommand } func zsetPopKeyMember(resp any) (any, any, bool) { diff --git a/proxy/command.go b/proxy/command.go index d3f7abb85..bd6697f46 100644 --- a/proxy/command.go +++ b/proxy/command.go @@ -151,7 +151,7 @@ var commandTable = map[string]CommandCategory{ "XAUTOCLAIM": CmdWrite, "ZADD": CmdWrite, "ZINCRBY": CmdWrite, - "ZREM": CmdWrite, + zremReplayCommand: CmdWrite, "ZREMRANGEBYSCORE": CmdWrite, "ZREMRANGEBYRANK": CmdWrite, "ZREMRANGEBYLEX": CmdWrite, diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index ca549991b..ddbf97ede 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -504,6 +504,11 @@ func (d *DualWriter) writeSecondaryPositiveIntWithOptions(sCtx context.Context, var resp any result := d.secondaryDo(sCtx, cmd, iArgs) resp, sErr = result.Result() + if shouldFallbackZRemFast(cmd, sErr) { + cmd = zremReplayCommand + iArgs = zremFastFallbackArgs(iArgs) + continue + } if ok, err := positiveIntReplayResult(resp, sErr); ok { elapsed := time.Since(start) d.metrics.CommandDuration.WithLabelValues(cmd, d.secondary.Name()).Observe(elapsed.Seconds()) @@ -865,20 +870,20 @@ func secondaryScriptTimeout(cfg ProxyConfig) time.Duration { } func (d *DualWriter) secondaryDo(ctx context.Context, cmd string, args []any) *redis.Cmd { - timeout := d.secondaryCommandReadTimeout(cmd) + timeout := d.secondaryCommandReadTimeout(ctx, cmd) if timeout > 0 { - if backend, ok := d.secondary.(blockingTimeoutBackend); ok { - return backend.DoWithTimeout(ctx, timeout, args...) + if backend, ok := d.secondary.(readTimeoutBackend); ok { + return backend.DoWithReadTimeout(ctx, timeout, args...) } } return d.secondary.Do(ctx, args...) } func (d *DualWriter) secondaryPipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { - timeout := d.secondaryPipelineReadTimeout(cmds) + timeout := d.secondaryPipelineReadTimeout(ctx, cmds) if timeout > 0 { - if backend, ok := d.secondary.(pipelineTimeoutBackend); ok { - results, err := backend.PipelineWithTimeout(ctx, timeout, cmds) + if backend, ok := d.secondary.(pipelineReadTimeoutBackend); ok { + results, err := backend.PipelineWithReadTimeout(ctx, timeout, cmds) if err != nil { return results, fmt.Errorf("secondary pipeline with timeout: %w", err) } @@ -892,18 +897,56 @@ func (d *DualWriter) secondaryPipeline(ctx context.Context, cmds [][]any) ([]*re return results, nil } -func (d *DualWriter) secondaryCommandReadTimeout(cmd string) time.Duration { +func (d *DualWriter) secondaryCommandReadTimeout(ctx context.Context, cmd string) time.Duration { if isRedisScriptCommandName(cmd) { - return secondaryScriptTimeout(d.cfg) + return secondaryReadTimeout(ctx, secondaryScriptTimeout(d.cfg)) } - return d.cfg.SecondaryTimeout + return secondaryReadTimeout(ctx, d.cfg.SecondaryTimeout) } -func (d *DualWriter) secondaryPipelineReadTimeout(cmds [][]any) time.Duration { +func (d *DualWriter) secondaryPipelineReadTimeout(ctx context.Context, cmds [][]any) time.Duration { if pipelineContainsScript(cmds) { - return secondaryScriptTimeout(d.cfg) + return secondaryReadTimeout(ctx, secondaryScriptTimeout(d.cfg)) + } + return secondaryReadTimeout(ctx, d.cfg.SecondaryTimeout) +} + +func secondaryReadTimeout(ctx context.Context, configured time.Duration) time.Duration { + if configured <= 0 { + return 0 + } + deadline, ok := ctx.Deadline() + if !ok { + return configured + } + remaining := time.Until(deadline) + if remaining <= 0 { + return 0 } - return d.cfg.SecondaryTimeout + if remaining < configured { + return remaining + } + return configured +} + +func shouldFallbackZRemFast(cmd string, err error) bool { + return strings.EqualFold(cmd, elasticKVZRemFastCommand) && isUnsupportedCommandError(err) +} + +func isUnsupportedCommandError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "unsupported command") || strings.Contains(msg, "unknown command") +} + +func zremFastFallbackArgs(args []any) []any { + out := append([]any(nil), args...) + if len(out) > 0 { + out[0] = zremReplayCommand + } + return out } func pipelineContainsScript(cmds [][]any) bool { diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index f15a0f7d0..24c020323 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -435,6 +435,27 @@ func (b *LeaderAwareRedisBackend) doWithTimeoutOnce(ctx context.Context, timeout return cli.WithTimeout(effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) } +func (b *LeaderAwareRedisBackend) DoWithReadTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { + cmd := b.doWithReadTimeoutOnce(ctx, timeout, args...) + switch { + case isElasticKVNotLeaderError(cmd.Err()): + b.RefreshLeaderNow(ctx) + case isLeaderRefreshTransportError(cmd.Err()): + b.TriggerRefresh() + } + return cmd +} + +func (b *LeaderAwareRedisBackend) doWithReadTimeoutOnce(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { + cli := b.currentClient() + if cli == nil { + cmd := redis.NewCmd(ctx, args...) + cmd.SetErr(ErrNoLeaderBackend) + return cmd + } + return cli.WithTimeout(timeout).Do(ctx, args...) +} + // Pipeline forwards a batch to the current leader. func (b *LeaderAwareRedisBackend) Pipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { return b.pipeline(ctx, 0, cmds) @@ -445,6 +466,10 @@ func (b *LeaderAwareRedisBackend) PipelineWithTimeout(ctx context.Context, timeo return b.pipeline(ctx, effectiveBlockingReadTimeout(timeout), cmds) } +func (b *LeaderAwareRedisBackend) PipelineWithReadTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { + return b.pipeline(ctx, timeout, cmds) +} + func (b *LeaderAwareRedisBackend) pipeline(ctx context.Context, readTimeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { cli := b.currentClient() if cli == nil { diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 6934e8629..b38a6d28b 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -569,6 +569,10 @@ func (b *timeoutCapturingBackend) DoWithTimeout(ctx context.Context, timeout tim return cmd } +func (b *timeoutCapturingBackend) DoWithReadTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { + return b.DoWithTimeout(ctx, timeout, args...) +} + func (b *timeoutCapturingBackend) Pipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { b.pipelineCalls++ b.pipelineCmds = clonePipelineArgs(cmds) @@ -596,6 +600,10 @@ func (b *timeoutCapturingBackend) PipelineWithTimeout(ctx context.Context, timeo return results, nil } +func (b *timeoutCapturingBackend) PipelineWithReadTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { + return b.PipelineWithTimeout(ctx, timeout, cmds) +} + func (b *timeoutCapturingBackend) Close() error { return nil } func (b *timeoutCapturingBackend) Name() string { return b.name } @@ -676,8 +684,8 @@ func TestDualWriter_Blocking_ReplaysBZPopAsZRem(t *testing.T) { secondaryName string expectedCmd string }{ - {name: "generic secondary BZPOPMIN", cmd: "BZPOPMIN", secondaryName: "secondary", expectedCmd: "ZREM"}, - {name: "generic secondary BZPOPMAX", cmd: "BZPOPMAX", secondaryName: "secondary", expectedCmd: "ZREM"}, + {name: "generic secondary BZPOPMIN", cmd: "BZPOPMIN", secondaryName: "secondary", expectedCmd: zremReplayCommand}, + {name: "generic secondary BZPOPMAX", cmd: "BZPOPMAX", secondaryName: "secondary", expectedCmd: zremReplayCommand}, {name: "elastickv secondary BZPOPMIN", cmd: "BZPOPMIN", secondaryName: "elastickv", expectedCmd: elasticKVZRemFastCommand}, {name: "elastickv secondary BZPOPMAX", cmd: "BZPOPMAX", secondaryName: "elastickv", expectedCmd: elasticKVZRemFastCommand}, } { @@ -756,6 +764,47 @@ func TestDualWriter_Blocking_RetriesBZPopReplayUntilRemoved(t *testing.T) { assert.Equal(t, 2, secondary.CallCount()) } +func TestDualWriter_Blocking_ZRemFastFallsBackToZRemWhenUnsupported(t *testing.T) { + primary := &timeoutCapturingBackend{ + name: "primary", + returnValue: []any{"queue", "job-1", "12.5"}, + } + secondary := newMockBackend("elastickv") + var calls []string + secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { + cmd := redis.NewCmd(ctx, args...) + name := fmt.Sprint(args[0]) + calls = append(calls, name) + if name == elasticKVZRemFastCommand { + cmd.SetErr(testRedisErr("ERR unsupported command 'ELASTICKV.ZREMFAST'")) + return cmd + } + cmd.SetVal(int64(1)) + return cmd + } + + metrics := newTestMetrics() + d := NewDualWriter( + primary, + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: time.Second, + SecondaryBlockingReplayConcurrency: 1, + }, + metrics, + newTestSentry(), + testLogger, + ) + + _, err := d.Blocking(context.Background(), "BZPOPMIN", [][]byte{[]byte("BZPOPMIN"), []byte("queue"), []byte("5")}) + assert.NoError(t, err) + d.Close() + + assert.Equal(t, []string{elasticKVZRemFastCommand, zremReplayCommand}, calls) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001) +} + func TestDualWriter_Blocking_BZPopReplayMissIsNotSecondaryWriteError(t *testing.T) { primary := &timeoutCapturingBackend{ name: "primary", @@ -785,7 +834,7 @@ func TestDualWriter_Blocking_BZPopReplayMissIsNotSecondaryWriteError(t *testing. assert.Equal(t, noEffectReplayRetryLimit(context.Background(), blockingReplayNoEffectRetryWindow)+1, secondary.CallCount()) assert.InDelta(t, 0, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001) assert.InDelta(t, 1, testutil.ToFloat64( - metrics.CommandTotal.WithLabelValues("ZREM", "secondary", "miss")), 0.001) + metrics.CommandTotal.WithLabelValues(zremReplayCommand, "secondary", "miss")), 0.001) } func TestDualWriter_Blocking_BZPopReplayShortTimeoutStillAttemptsZRem(t *testing.T) { @@ -817,7 +866,7 @@ func TestDualWriter_Blocking_BZPopReplayShortTimeoutStillAttemptsZRem(t *testing assert.GreaterOrEqual(t, secondary.CallCount(), 1) assert.InDelta(t, 0, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001) assert.InDelta(t, 1, testutil.ToFloat64( - metrics.CommandTotal.WithLabelValues("ZREM", "secondary", "miss")), 0.001) + metrics.CommandTotal.WithLabelValues(zremReplayCommand, "secondary", "miss")), 0.001) } func TestDualWriter_Blocking_DoesNotReplayNoEffectBZPop(t *testing.T) { @@ -1322,6 +1371,36 @@ func TestDualWriter_SecondaryUsesPerCommandSocketTimeouts(t *testing.T) { assert.Equal(t, 5*time.Minute, secondary.timeout) } +func TestDualWriter_SecondaryReadTimeoutUsesRemainingAsyncDeadline(t *testing.T) { + secondary := &timeoutCapturingBackend{name: "secondary", returnValue: "OK"} + d := NewDualWriter( + newMockBackend("primary"), + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: 5 * time.Minute, + }, + newTestMetrics(), + newTestSentry(), + testLogger, + ) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + d.writeSecondary(ctx, "EVALSHA", []any{[]byte("EVALSHA"), []byte("deadbeef"), []byte("0")}) + assert.Equal(t, 1, secondary.doWithCalls) + assert.Greater(t, secondary.timeout, time.Duration(0)) + assert.LessOrEqual(t, secondary.timeout, 50*time.Millisecond) + + pipelineCtx, pipelineCancel := context.WithTimeout(context.Background(), 80*time.Millisecond) + defer pipelineCancel() + d.writeSecondaryPipeline(pipelineCtx, [][]any{{"MULTI"}, {"EVALSHA", "deadbeef", "0"}, {"EXEC"}}) + assert.Equal(t, 1, secondary.pipelineWithCalls) + assert.Greater(t, secondary.pipelineTimeout, time.Duration(0)) + assert.LessOrEqual(t, secondary.pipelineTimeout, 80*time.Millisecond) +} + func TestDualWriter_TxnReplayWithScriptUsesScriptQueueAndTimeout(t *testing.T) { secondary := &timeoutCapturingBackend{name: "secondary", returnValue: "OK"} metrics := newTestMetrics() @@ -1349,7 +1428,8 @@ func TestDualWriter_TxnReplayWithScriptUsesScriptQueueAndTimeout(t *testing.T) { d.Close() assert.Equal(t, 1, secondary.pipelineWithCalls) - assert.Equal(t, 5*time.Minute, secondary.pipelineTimeout) + assert.Greater(t, secondary.pipelineTimeout, time.Duration(0)) + assert.LessOrEqual(t, secondary.pipelineTimeout, 5*time.Minute) assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDropsByQueue.WithLabelValues(asyncQueueScript, asyncDropQueueFull)), 0.001) } @@ -1379,7 +1459,8 @@ func TestDualWriter_TxnReplayWithoutScriptUsesWriteQueueAndTimeout(t *testing.T) d.Close() assert.Equal(t, 1, secondary.pipelineWithCalls) - assert.Equal(t, 30*time.Second, secondary.pipelineTimeout) + assert.Greater(t, secondary.pipelineTimeout, time.Duration(0)) + assert.LessOrEqual(t, secondary.pipelineTimeout, 30*time.Second) } func TestDualWriter_ExpiredQueuedWorkDoesNotWaitForSemaphore(t *testing.T) { From 8f100ddf7880d6b9d1b90c1b288345b28538cadf Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 20:18:05 +0900 Subject: [PATCH 14/18] redis-proxy: keep async replay writes bounded --- proxy/backend.go | 21 ++++++++++++--- proxy/dualwrite.go | 31 +++++++++++++++------- proxy/leader_aware_backend.go | 6 ++--- proxy/proxy_test.go | 50 +++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 17 deletions(-) diff --git a/proxy/backend.go b/proxy/backend.go index d4818cd18..2d3372f63 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -125,11 +125,11 @@ func (b *RedisBackend) Do(ctx context.Context, args ...any) *redis.Cmd { // This is used for blocking commands whose wait time exceeds the backend's // default read timeout. func (b *RedisBackend) DoWithTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { - return b.client.WithTimeout(effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) + return redisClientWithReadTimeout(b.client, effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) } func (b *RedisBackend) DoWithReadTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { - return b.client.WithTimeout(timeout).Do(ctx, args...) + return redisClientWithReadTimeout(b.client, timeout).Do(ctx, args...) } func effectiveBlockingReadTimeout(timeout time.Duration) time.Duration { @@ -146,11 +146,24 @@ func (b *RedisBackend) Pipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd // PipelineWithTimeout executes a pipeline using a per-call socket timeout // override for async replay classes whose deadline exceeds the backend default. func (b *RedisBackend) PipelineWithTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { - return b.pipeline(ctx, b.client.WithTimeout(effectiveBlockingReadTimeout(timeout)), cmds) + return b.pipeline(ctx, redisClientWithReadTimeout(b.client, effectiveBlockingReadTimeout(timeout)), cmds) } func (b *RedisBackend) PipelineWithReadTimeout(ctx context.Context, timeout time.Duration, cmds [][]any) ([]*redis.Cmd, error) { - return b.pipeline(ctx, b.client.WithTimeout(timeout), cmds) + return b.pipeline(ctx, redisClientWithReadTimeout(b.client, timeout), cmds) +} + +func redisClientWithReadTimeout(client *redis.Client, timeout time.Duration) *redis.Client { + if client == nil || timeout <= 0 { + return client + } + clone := client.WithTimeout(timeout) + // go-redis WithTimeout intentionally sets both ReadTimeout and + // WriteTimeout. This path only extends read-side waits for blocking + // commands and async replays; keep socket writes bounded by the backend's + // configured write timeout. + clone.Options().WriteTimeout = client.Options().WriteTimeout + return clone } func (b *RedisBackend) pipeline(ctx context.Context, client *redis.Client, cmds [][]any) ([]*redis.Cmd, error) { diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index ddbf97ede..08c29d0fe 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -870,7 +870,12 @@ func secondaryScriptTimeout(cfg ProxyConfig) time.Duration { } func (d *DualWriter) secondaryDo(ctx context.Context, cmd string, args []any) *redis.Cmd { - timeout := d.secondaryCommandReadTimeout(ctx, cmd) + timeout, err := d.secondaryCommandReadTimeout(ctx, cmd) + if err != nil { + result := redis.NewCmd(ctx, args...) + result.SetErr(err) + return result + } if timeout > 0 { if backend, ok := d.secondary.(readTimeoutBackend); ok { return backend.DoWithReadTimeout(ctx, timeout, args...) @@ -880,7 +885,10 @@ func (d *DualWriter) secondaryDo(ctx context.Context, cmd string, args []any) *r } func (d *DualWriter) secondaryPipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { - timeout := d.secondaryPipelineReadTimeout(ctx, cmds) + timeout, err := d.secondaryPipelineReadTimeout(ctx, cmds) + if err != nil { + return nil, err + } if timeout > 0 { if backend, ok := d.secondary.(pipelineReadTimeoutBackend); ok { results, err := backend.PipelineWithReadTimeout(ctx, timeout, cmds) @@ -897,36 +905,39 @@ func (d *DualWriter) secondaryPipeline(ctx context.Context, cmds [][]any) ([]*re return results, nil } -func (d *DualWriter) secondaryCommandReadTimeout(ctx context.Context, cmd string) time.Duration { +func (d *DualWriter) secondaryCommandReadTimeout(ctx context.Context, cmd string) (time.Duration, error) { if isRedisScriptCommandName(cmd) { return secondaryReadTimeout(ctx, secondaryScriptTimeout(d.cfg)) } return secondaryReadTimeout(ctx, d.cfg.SecondaryTimeout) } -func (d *DualWriter) secondaryPipelineReadTimeout(ctx context.Context, cmds [][]any) time.Duration { +func (d *DualWriter) secondaryPipelineReadTimeout(ctx context.Context, cmds [][]any) (time.Duration, error) { if pipelineContainsScript(cmds) { return secondaryReadTimeout(ctx, secondaryScriptTimeout(d.cfg)) } return secondaryReadTimeout(ctx, d.cfg.SecondaryTimeout) } -func secondaryReadTimeout(ctx context.Context, configured time.Duration) time.Duration { +func secondaryReadTimeout(ctx context.Context, configured time.Duration) (time.Duration, error) { if configured <= 0 { - return 0 + return 0, nil } deadline, ok := ctx.Deadline() if !ok { - return configured + return configured, nil } remaining := time.Until(deadline) if remaining <= 0 { - return 0 + if err := ctx.Err(); err != nil { + return 0, fmt.Errorf("secondary read timeout deadline expired: %w", err) + } + return 0, fmt.Errorf("secondary read timeout deadline expired: %w", context.DeadlineExceeded) } if remaining < configured { - return remaining + return remaining, nil } - return configured + return configured, nil } func shouldFallbackZRemFast(cmd string, err error) bool { diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index 24c020323..30ea61b2a 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -432,7 +432,7 @@ func (b *LeaderAwareRedisBackend) doWithTimeoutOnce(ctx context.Context, timeout cmd.SetErr(ErrNoLeaderBackend) return cmd } - return cli.WithTimeout(effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) + return redisClientWithReadTimeout(cli, effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) } func (b *LeaderAwareRedisBackend) DoWithReadTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { @@ -453,7 +453,7 @@ func (b *LeaderAwareRedisBackend) doWithReadTimeoutOnce(ctx context.Context, tim cmd.SetErr(ErrNoLeaderBackend) return cmd } - return cli.WithTimeout(timeout).Do(ctx, args...) + return redisClientWithReadTimeout(cli, timeout).Do(ctx, args...) } // Pipeline forwards a batch to the current leader. @@ -476,7 +476,7 @@ func (b *LeaderAwareRedisBackend) pipeline(ctx context.Context, readTimeout time return nil, ErrNoLeaderBackend } if readTimeout > 0 { - cli = cli.WithTimeout(readTimeout) + cli = redisClientWithReadTimeout(cli, readTimeout) } pipe := cli.Pipeline() results := make([]*redis.Cmd, len(cmds)) diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index b38a6d28b..4e1ecaffc 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -1401,6 +1401,38 @@ func TestDualWriter_SecondaryReadTimeoutUsesRemainingAsyncDeadline(t *testing.T) assert.LessOrEqual(t, secondary.pipelineTimeout, 80*time.Millisecond) } +func TestDualWriter_ExpiredSecondaryReadDeadlineSkipsDispatch(t *testing.T) { + secondary := &timeoutCapturingBackend{name: "secondary", returnValue: "OK"} + d := NewDualWriter( + newMockBackend("primary"), + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: 30 * time.Second, + SecondaryScriptTimeout: 5 * time.Minute, + }, + newTestMetrics(), + newTestSentry(), + testLogger, + ) + t.Cleanup(d.Close) + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Millisecond)) + defer cancel() + + cmd := d.secondaryDo(ctx, "EVALSHA", []any{"EVALSHA", "deadbeef", "0"}) + _, err := cmd.Result() + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Zero(t, secondary.doCalls) + assert.Zero(t, secondary.doWithCalls) + + results, err := d.secondaryPipeline(ctx, [][]any{{"MULTI"}, {"EVALSHA", "deadbeef", "0"}, {"EXEC"}}) + assert.Nil(t, results) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Zero(t, secondary.pipelineCalls) + assert.Zero(t, secondary.pipelineWithCalls) +} + func TestDualWriter_TxnReplayWithScriptUsesScriptQueueAndTimeout(t *testing.T) { secondary := &timeoutCapturingBackend{name: "secondary", returnValue: "OK"} metrics := newTestMetrics() @@ -1671,6 +1703,24 @@ func TestEffectiveBlockingReadTimeout(t *testing.T) { assert.Equal(t, 11*time.Second, effectiveBlockingReadTimeout(time.Second)) } +func TestRedisClientWithReadTimeoutPreservesWriteTimeout(t *testing.T) { + client := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + ReadTimeout: 3 * time.Second, + WriteTimeout: 3 * time.Second, + }) + t.Cleanup(func() { + assert.NoError(t, client.Close()) + }) + + clone := redisClientWithReadTimeout(client, 5*time.Minute) + assert.Equal(t, 5*time.Minute, clone.Options().ReadTimeout) + assert.Equal(t, 3*time.Second, clone.Options().WriteTimeout) + assert.Equal(t, 3*time.Second, client.Options().ReadTimeout) + assert.Equal(t, 3*time.Second, client.Options().WriteTimeout) + assert.Same(t, client, redisClientWithReadTimeout(client, 0)) +} + // ========== Pipeline error handling tests ========== func TestPipeline_TransportError(t *testing.T) { From ddfffb760ed0538df62aaafd0c12acf39c942150 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 21:37:13 +0900 Subject: [PATCH 15/18] proxy: refresh leader for timed redis commands --- proxy/leader_aware_backend.go | 2 +- proxy/leader_aware_backend_test.go | 43 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index 4f9ee2b7f..2889e4f13 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -496,7 +496,7 @@ func (b *LeaderAwareRedisBackend) DoWithReadTimeout(ctx context.Context, timeout } func (b *LeaderAwareRedisBackend) doWithReadTimeoutOnce(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { - cli := b.currentClient() + cli := b.currentClientOrRefresh(ctx) if cli == nil { cmd := redis.NewCmd(ctx, args...) cmd.SetErr(ErrNoLeaderBackend) diff --git a/proxy/leader_aware_backend_test.go b/proxy/leader_aware_backend_test.go index 5d40cefd7..221e81655 100644 --- a/proxy/leader_aware_backend_test.go +++ b/proxy/leader_aware_backend_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -332,6 +333,48 @@ func TestLeaderAwareRedisBackend_ScriptNotLeaderRefreshesWithoutReplay(t *testin require.Equal(t, int64(0), nodeB.commands.Load(), "script must not be replayed after refresh") } +func TestLeaderAwareRedisBackend_DoWithReadTimeoutWaitsForInitialLeaderDiscovery(t *testing.T) { + node := newFakeElasticKVNode(t) + node.SetLeader(node.addr) + gate := make(chan struct{}) + node.SetInfoGate(gate) + + backend := NewLeaderAwareRedisBackendWithInterval( + []string{node.addr}, + "elastickv", + DefaultBackendOptions(), + time.Hour, 500*time.Millisecond, + testLogger, + ) + t.Cleanup(func() { _ = backend.Close() }) + + result := make(chan *redis.Cmd, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + result <- backend.DoWithReadTimeout(ctx, 100*time.Millisecond, "SET", "k", "v") + }() + + require.Eventually(t, func() bool { + return node.infoCalls.Load() > 0 + }, time.Second, 10*time.Millisecond) + select { + case cmd := <-result: + require.Failf(t, "command returned before leader discovery completed", "err=%v", cmd.Err()) + case <-time.After(50 * time.Millisecond): + } + + close(gate) + select { + case cmd := <-result: + require.NoError(t, cmd.Err()) + case <-time.After(2 * time.Second): + t.Fatal("command did not finish after leader discovery completed") + } + require.Equal(t, node.addr, backend.CurrentLeader()) + require.Equal(t, int64(1), node.commands.Load()) +} + func TestLeaderAwareRedisBackend_DoesNotRetryScriptNotLeaderRedisError(t *testing.T) { node := newFakeElasticKVNode(t) node.SetLeader(node.addr) From 40370e4e57f9c8e11b6258f2f722fe1a0bb55aed Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 21:57:18 +0900 Subject: [PATCH 16/18] test: stabilize dynamodb leader health check --- adapter/dynamodb_test.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/adapter/dynamodb_test.go b/adapter/dynamodb_test.go index 8c071303c..ab22af515 100644 --- a/adapter/dynamodb_test.go +++ b/adapter/dynamodb_test.go @@ -268,22 +268,23 @@ func TestDynamoDB_LeaderHealthz(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - req, err := http.NewRequestWithContext( - context.Background(), - http.MethodGet, - "http://"+tc.addr+dynamoLeaderHealthPath, - nil, - ) - require.NoError(t, err) - - resp, err := http.DefaultClient.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - - require.Equal(t, tc.status, resp.StatusCode) - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - require.Equal(t, tc.body, string(body)) + require.Eventually(t, func() bool { + req, err := http.NewRequestWithContext( + context.Background(), + http.MethodGet, + "http://"+tc.addr+dynamoLeaderHealthPath, + nil, + ) + require.NoError(t, err) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + return resp.StatusCode == tc.status && string(body) == tc.body + }, leaderChurnRetryTimeout, leaderChurnRetryInterval) }) } } From cf09c1bd443a3889a858b60d28be061dcf9a1899 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 22:27:58 +0900 Subject: [PATCH 17/18] test: bound dynamodb health check retries --- adapter/dynamodb_test.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/adapter/dynamodb_test.go b/adapter/dynamodb_test.go index ab22af515..be86c007a 100644 --- a/adapter/dynamodb_test.go +++ b/adapter/dynamodb_test.go @@ -269,20 +269,29 @@ func TestDynamoDB_LeaderHealthz(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { require.Eventually(t, func() bool { + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + req, err := http.NewRequestWithContext( - context.Background(), + ctx, http.MethodGet, "http://"+tc.addr+dynamoLeaderHealthPath, nil, ) - require.NoError(t, err) + if err != nil { + return false + } resp, err := http.DefaultClient.Do(req) - require.NoError(t, err) + if err != nil { + return false + } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) - require.NoError(t, err) + if err != nil { + return false + } return resp.StatusCode == tc.status && string(body) == tc.body }, leaderChurnRetryTimeout, leaderChurnRetryInterval) }) From 424e35d29e68dcd023b0995d5bd3c245403dbbd7 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 23:13:21 +0900 Subject: [PATCH 18/18] redis-proxy: preserve blocking replay deadlines --- proxy/backend.go | 13 ++++++- proxy/dualwrite.go | 2 ++ proxy/leader_aware_backend.go | 2 +- proxy/proxy_test.go | 64 +++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) diff --git a/proxy/backend.go b/proxy/backend.go index 84460097c..ca42ba209 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -127,7 +127,7 @@ func (b *RedisBackend) Do(ctx context.Context, args ...any) *redis.Cmd { // This is used for blocking commands whose wait time exceeds the backend's // default read timeout. func (b *RedisBackend) DoWithTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { - return redisClientWithReadTimeout(b.client, effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) + return redisClientWithBlockingReadTimeout(b.client, timeout).Do(ctx, args...) } func (b *RedisBackend) DoWithReadTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { @@ -159,6 +159,17 @@ func redisClientWithReadTimeout(client *redis.Client, timeout time.Duration) *re if client == nil || timeout <= 0 { return client } + return redisClientWithReadTimeoutValue(client, timeout) +} + +func redisClientWithBlockingReadTimeout(client *redis.Client, timeout time.Duration) *redis.Client { + if client == nil { + return client + } + return redisClientWithReadTimeoutValue(client, effectiveBlockingReadTimeout(timeout)) +} + +func redisClientWithReadTimeoutValue(client *redis.Client, timeout time.Duration) *redis.Client { clone := client.WithTimeout(timeout) // go-redis WithTimeout intentionally sets both ReadTimeout and // WriteTimeout. This path only extends read-side waits for blocking diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index 463c47cea..cbb480b23 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -511,6 +511,8 @@ func (d *DualWriter) writeSecondaryPositiveIntWithOptions(sCtx context.Context, if shouldFallbackZRemFast(cmd, sErr) { cmd = zremReplayCommand iArgs = zremFastFallbackArgs(iArgs) + attempt = -1 + backoff = compactedRetryInitialBackoff continue } if ok, err := positiveIntReplayResult(resp, sErr); ok { diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index 2889e4f13..a71169012 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -481,7 +481,7 @@ func (b *LeaderAwareRedisBackend) doWithTimeoutOnce(ctx context.Context, timeout cmd.SetErr(ErrNoLeaderBackend) return cmd } - return redisClientWithReadTimeout(cli, effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) + return redisClientWithBlockingReadTimeout(cli, timeout).Do(ctx, args...) } func (b *LeaderAwareRedisBackend) DoWithReadTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index dda2268b9..a45157b07 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -834,6 +834,52 @@ func TestDualWriter_Blocking_ZRemFastFallsBackToZRemWhenUnsupported(t *testing.T assert.InDelta(t, 0, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001) } +func TestDualWriter_Blocking_ZRemFastFallbackResetsNoEffectRetryBudget(t *testing.T) { + primary := &timeoutCapturingBackend{ + name: "primary", + returnValue: []any{"queue", "job-1", "12.5"}, + } + secondary := newMockBackend("elastickv") + var zremCalls atomic.Int32 + secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { + cmd := redis.NewCmd(ctx, args...) + switch name := fmt.Sprint(args[0]); name { + case elasticKVZRemFastCommand: + cmd.SetErr(testRedisErr("ERR unsupported command 'ELASTICKV.ZREMFAST'")) + case zremReplayCommand: + zremCalls.Add(1) + cmd.SetVal(int64(0)) + default: + cmd.SetErr(fmt.Errorf("unexpected replay command %s", name)) + } + return cmd + } + + metrics := newTestMetrics() + d := NewDualWriter( + primary, + secondary, + ProxyConfig{ + Mode: ModeDualWrite, + SecondaryTimeout: 2 * time.Second, + SecondaryBlockingReplayConcurrency: 1, + }, + metrics, + newTestSentry(), + testLogger, + ) + + _, err := d.Blocking(context.Background(), "BZPOPMIN", [][]byte{[]byte("BZPOPMIN"), []byte("queue"), []byte("5")}) + assert.NoError(t, err) + d.Close() + + limit := noEffectReplayRetryLimit(context.Background(), blockingReplayNoEffectRetryWindow) + assert.EqualValues(t, limit+1, zremCalls.Load()) + assert.Equal(t, limit+2, secondary.CallCount()) + assert.InDelta(t, 1, testutil.ToFloat64( + metrics.CommandTotal.WithLabelValues(zremReplayCommand, "elastickv", "miss")), 0.001) +} + func TestDualWriter_Blocking_BZPopReplayMissIsNotSecondaryWriteError(t *testing.T) { primary := &timeoutCapturingBackend{ name: "primary", @@ -1750,6 +1796,24 @@ func TestRedisClientWithReadTimeoutPreservesWriteTimeout(t *testing.T) { assert.Same(t, client, redisClientWithReadTimeout(client, 0)) } +func TestRedisClientWithBlockingReadTimeoutZeroDisablesReadDeadline(t *testing.T) { + client := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:6379", + ReadTimeout: 3 * time.Second, + WriteTimeout: 3 * time.Second, + }) + t.Cleanup(func() { + assert.NoError(t, client.Close()) + }) + + clone := redisClientWithBlockingReadTimeout(client, 0) + assert.NotSame(t, client, clone) + assert.Equal(t, time.Duration(0), clone.Options().ReadTimeout) + assert.Equal(t, 3*time.Second, clone.Options().WriteTimeout) + assert.Equal(t, 3*time.Second, client.Options().ReadTimeout) + assert.Equal(t, 3*time.Second, client.Options().WriteTimeout) +} + // ========== Pipeline error handling tests ========== func TestPipeline_TransportError(t *testing.T) {