From 8fbb93d197e4b40144692762db7226d9c0e9f6df Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Mon, 13 Apr 2026 18:21:42 -0700 Subject: [PATCH] Expose Redis pool metrics --- cmd/asb-api/observability.go | 20 +++++++++++--- cmd/asb-api/observability_test.go | 44 +++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +-- internal/bootstrap/service.go | 29 +++++++++++++------- 5 files changed, 82 insertions(+), 17 deletions(-) diff --git a/cmd/asb-api/observability.go b/cmd/asb-api/observability.go index 47f37ca..d93820e 100644 --- a/cmd/asb-api/observability.go +++ b/cmd/asb-api/observability.go @@ -27,10 +27,22 @@ func newObservedHandler(logger *slog.Logger, metrics *observability.Metrics, nex } func registerRuntimeMetrics(runtime *bootstrap.ServiceRuntime, registerer prometheus.Registerer) error { - if runtime == nil || runtime.DBStats == nil { + if runtime == nil { return nil } - return observability.RegisterDBStats("asb", runtime.DBStats, observability.DBStatsOptions{ - Registerer: registerer, - }) + if runtime.DBStats != nil { + if err := observability.RegisterDBStats("asb", runtime.DBStats, observability.DBStatsOptions{ + Registerer: registerer, + }); err != nil { + return err + } + } + if runtime.RedisStats != nil { + if err := observability.RegisterRedisPoolStats("asb", runtime.RedisStats, observability.RedisStatsOptions{ + Registerer: registerer, + }); err != nil { + return err + } + } + return nil } diff --git a/cmd/asb-api/observability_test.go b/cmd/asb-api/observability_test.go index 29f8da1..f9c85b5 100644 --- a/cmd/asb-api/observability_test.go +++ b/cmd/asb-api/observability_test.go @@ -12,6 +12,7 @@ import ( "github.com/evalops/asb/internal/bootstrap" "github.com/evalops/service-runtime/observability" "github.com/prometheus/client_golang/prometheus" + goredis "github.com/redis/go-redis/v9" ) func TestNewObservedHandlerServesMetrics(t *testing.T) { @@ -133,6 +134,49 @@ func TestRegisterRuntimeMetricsRegistersDBStats(t *testing.T) { } } +func TestRegisterRuntimeMetricsRegistersRedisStats(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + runtime := &bootstrap.ServiceRuntime{ + RedisStats: func() *goredis.PoolStats { + return &goredis.PoolStats{ + Hits: 11, + TotalConns: 5, + IdleConns: 2, + PendingRequests: 1, + } + }, + } + + if err := registerRuntimeMetrics(runtime, registry); err != nil { + t.Fatalf("registerRuntimeMetrics() error = %v", err) + } + + metrics, err := observability.NewMetrics("asb", observability.MetricsOptions{ + Registerer: registry, + Gatherer: registry, + }) + if err != nil { + t.Fatalf("NewMetrics() error = %v", err) + } + + handler := newObservedHandler(discardLogger(), metrics, http.NewServeMux()) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + body := recorder.Body.String() + if !strings.Contains(body, "asb_redis_pool_total_connections 5") { + t.Fatalf("metrics body = %q, want Redis total connection gauge", body) + } + if !strings.Contains(body, "asb_redis_pool_idle_connections 2") { + t.Fatalf("metrics body = %q, want Redis idle connection gauge", body) + } + if !strings.Contains(body, "asb_redis_pool_hits_total 11") { + t.Fatalf("metrics body = %q, want Redis hits counter", body) + } +} + func discardLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } diff --git a/go.mod b/go.mod index cdd10f3..2bdb4e4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( connectrpc.com/connect v1.19.1 github.com/alicebob/miniredis/v2 v2.37.0 github.com/evalops/proto v0.0.0-20260414000509-37b2bf5d6244 - github.com/evalops/service-runtime v0.1.16 + github.com/evalops/service-runtime v0.1.17 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/jackc/pgx/v5 v5.9.1 github.com/pashagolub/pgxmock/v4 v4.9.0 diff --git a/go.sum b/go.sum index 5ad6944..db7d11b 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/evalops/proto v0.0.0-20260414000509-37b2bf5d6244 h1:MjoKPIxG/OisQLDqLvbvpleCAkPgr2InTrK4Q0D9F1o= github.com/evalops/proto v0.0.0-20260414000509-37b2bf5d6244/go.mod h1:EXB8IcqMaV58Tt0w2GaQia3YOwzrEvnIWFZetHX+IxE= -github.com/evalops/service-runtime v0.1.16 h1:pgxFcQep8t7p6sanvTWvu9nxy0lKeaeOZlDqHvIAlQY= -github.com/evalops/service-runtime v0.1.16/go.mod h1:ZAB9GcnbFaFUq+X5TsoMUUth6KnAVR6uVHMKanXtQTY= +github.com/evalops/service-runtime v0.1.17 h1:Elnn2Z+qLoxweauNikJQnQKt/0smEEdzOc0qQCqxo5g= +github.com/evalops/service-runtime v0.1.17/go.mod h1:ZAB9GcnbFaFUq+X5TsoMUUth6KnAVR6uVHMKanXtQTY= github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= diff --git a/internal/bootstrap/service.go b/internal/bootstrap/service.go index 910a601..94d90cc 100644 --- a/internal/bootstrap/service.go +++ b/internal/bootstrap/service.go @@ -48,10 +48,11 @@ type serviceOptions struct { type readinessProbe func(context.Context) error type ServiceRuntime struct { - Service *app.Service - Cleanup func() - Health *HealthChecker - DBStats func() sql.DBStats + Service *app.Service + Cleanup func() + Health *HealthChecker + DBStats func() sql.DBStats + RedisStats func() *goredis.PoolStats } type HealthChecker struct { @@ -102,7 +103,7 @@ func NewServiceRuntime(ctx context.Context, logger *slog.Logger, options ...Serv if err != nil { return nil, err } - runtimeStore, cleanupRuntime, redisProbe, err := newRuntimeStore(ctx) + runtimeStore, cleanupRuntime, redisProbe, redisStats, err := newRuntimeStore(ctx) if err != nil { cleanupRepository() return nil, err @@ -268,7 +269,8 @@ func NewServiceRuntime(ctx context.Context, logger *slog.Logger, options ...Serv redisProbe: redisProbe, sessionTokensReady: sessionTokens != nil, }, - DBStats: dbStats, + DBStats: dbStats, + RedisStats: redisStats, }, nil } @@ -460,7 +462,7 @@ func newRepository(ctx context.Context) (core.Repository, func(), readinessProbe return memstore.NewRepository(), func() {}, nil, nil, nil } -func newRuntimeStore(ctx context.Context) (core.RuntimeStore, func(), readinessProbe, error) { +func newRuntimeStore(ctx context.Context) (core.RuntimeStore, func(), readinessProbe, func() *goredis.PoolStats, error) { if addr := os.Getenv("ASB_REDIS_ADDR"); addr != "" { client := goredis.NewClient(&goredis.Options{ Addr: addr, @@ -468,13 +470,13 @@ func newRuntimeStore(ctx context.Context) (core.RuntimeStore, func(), readinessP DB: 0, }) if err := client.Ping(ctx).Err(); err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } return redisstore.NewRuntimeStore(client), func() { _ = client.Close() }, func(ctx context.Context) error { return client.Ping(ctx).Err() - }, nil + }, redisPoolStats(client), nil } - return memstore.NewRuntimeStore(), func() {}, nil, nil + return memstore.NewRuntimeStore(), func() {}, nil, nil, nil } func pgxPoolDBStats(pool *pgxpool.Pool) func() sql.DBStats { @@ -492,6 +494,13 @@ func pgxPoolDBStats(pool *pgxpool.Pool) func() sql.DBStats { } } +func redisPoolStats(client goredis.UniversalClient) func() *goredis.PoolStats { + if client == nil { + return nil + } + return client.PoolStats +} + func loadPublicKey(path string) (any, error) { contents, err := os.ReadFile(path) if err != nil {