From f031740a2d741d2e8aaa745f958362afb9ab4264 Mon Sep 17 00:00:00 2001 From: jcameron Date: Fri, 24 Jul 2026 15:59:24 -0300 Subject: [PATCH] feat(vanity-gateway): add per-model shadow drop metric Expose dropped shadow dispatches by target model and bounded reason, with zero-initialized series for configured targets. Refs: #428 --- .../vanity-gateway/MODULE.bazel | 3 +- .../vanity-gateway/README.md | 5 + .../vanity-gateway/gateway/BUILD.bazel | 7 ++ .../vanity-gateway/gateway/gateway.go | 3 + .../vanity-gateway/gateway/openai_director.go | 1 + .../vanity-gateway/gateway/shadow.go | 2 + .../vanity-gateway/gateway/shadow_metrics.go | 92 ++++++++++++++++ .../gateway/shadow_metrics_test.go | 100 ++++++++++++++++++ .../vanity-gateway/go.mod | 2 +- 9 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics.go create mode 100644 src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics_test.go diff --git a/src/invocation-plane-services/vanity-gateway/MODULE.bazel b/src/invocation-plane-services/vanity-gateway/MODULE.bazel index bbc3620f3..975a1cecc 100644 --- a/src/invocation-plane-services/vanity-gateway/MODULE.bazel +++ b/src/invocation-plane-services/vanity-gateway/MODULE.bazel @@ -101,8 +101,9 @@ use_repo( "io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_prometheus", - "io_opentelemetry_go_otel_sdk_metric", + "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", + "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "org_golang_google_grpc", "org_golang_x_exp", diff --git a/src/invocation-plane-services/vanity-gateway/README.md b/src/invocation-plane-services/vanity-gateway/README.md index 5e58745be..77c0553b3 100644 --- a/src/invocation-plane-services/vanity-gateway/README.md +++ b/src/invocation-plane-services/vanity-gateway/README.md @@ -369,6 +369,11 @@ curl -v -H 'Host: api.example.com' localhost:10081/health curl -v localhost:10083/metrics ``` +`nvcf_ai_api_gateway_shadow_requests_dropped_total` counts shadow dispatches +dropped before replay. The `openai_model_name` label identifies the shadow +target. The `reason` label is one of `body_read_error`, `body_rewrite_error`, +or `concurrency_limit`. + ## Running a Local OpenAI-compatible Model Use this only when validating OpenAI-compatible request bodies against a local model server. diff --git a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel index 926b9b845..ca8f7790d 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel +++ b/src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "health.go", "openai_director.go", "shadow.go", + "shadow_metrics.go", "tracing_attrs.go", "vanity_director.go", "version.go", @@ -32,8 +33,11 @@ go_library( "@com_github_nvidia_nvcf_go//pkg/nvkit/tracing", "@com_github_valyala_bytebufferpool//:bytebufferpool", "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:otelhttp", + "@io_opentelemetry_go_otel//:otel", "@io_opentelemetry_go_otel//attribute", "@io_opentelemetry_go_otel//propagation", + "@io_opentelemetry_go_otel_metric//:metric", + "@io_opentelemetry_go_otel_metric//noop", "@io_opentelemetry_go_otel_trace//:trace", "@org_golang_google_grpc//:grpc", "@org_golang_x_exp//maps", @@ -56,6 +60,7 @@ go_test( "gateway_test.go", "h2_test.go", "openai_director_test.go", + "shadow_metrics_test.go", "shadow_test.go", "shadow_tracing_test.go", "vanity_director_test.go", @@ -74,6 +79,8 @@ go_test( "@io_opentelemetry_go_otel//codes", "@io_opentelemetry_go_otel_sdk//trace", "@io_opentelemetry_go_otel_sdk//trace/tracetest", + "@io_opentelemetry_go_otel_sdk_metric//:metric", + "@io_opentelemetry_go_otel_sdk_metric//metricdata", "@io_opentelemetry_go_otel_trace//:trace", "@org_uber_go_zap//:zap", "@org_uber_go_zap//zaptest/observer", diff --git a/src/invocation-plane-services/vanity-gateway/gateway/gateway.go b/src/invocation-plane-services/vanity-gateway/gateway/gateway.go index 9c9977789..a9315f296 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/gateway.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/gateway.go @@ -104,6 +104,9 @@ func NewNVCFGateway(logger *logs.ZapLogger, config Config) (*NVCFGateway, error) if err := middleware.SetupHTTPMetrics(); err != nil { return nil, fmt.Errorf("failed to setup HTTP metrics: %w", err) } + if err := setupShadowMetrics(); err != nil { + return nil, fmt.Errorf("failed to setup shadow metrics: %w", err) + } mappings, err := gatewayConfig.SetupConfigWithConfigPath(config.MappingPath) if err != nil { diff --git a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go index 20bb1da55..9d0106aed 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/openai_director.go @@ -236,6 +236,7 @@ func buildModelMapping( if entry.OutgoingPathOverride != "" { pathOverride = &entry.OutgoingPathOverride } + initializeShadowDropMetrics(entry.ShadowModelNames) modelNameToNVCFUrl[modelName] = FunctionInfo{ functionId: entry.FunctionId, functionVersionId: entry.FunctionVersionId, diff --git a/src/invocation-plane-services/vanity-gateway/gateway/shadow.go b/src/invocation-plane-services/vanity-gateway/gateway/shadow.go index 97dd1d155..35beae4ac 100644 --- a/src/invocation-plane-services/vanity-gateway/gateway/shadow.go +++ b/src/invocation-plane-services/vanity-gateway/gateway/shadow.go @@ -123,6 +123,8 @@ func shadowContext(req *http.Request, cancelOnClientDisconnect bool) (context.Co } func recordShadowDispatchSummary(ctx context.Context, targetModels []string, dispatchedCount int, droppedCount int, droppedReasons []string, droppedTargetModels []string) { + recordShadowDropMetrics(ctx, droppedReasons, droppedTargetModels) + span := trace.SpanFromContext(ctx) attrs := []attribute.KeyValue{ traceAttrShadowDispatched.Bool(dispatchedCount > 0), diff --git a/src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics.go b/src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics.go new file mode 100644 index 000000000..ed3934a7d --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics.go @@ -0,0 +1,92 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" + metricnoop "go.opentelemetry.io/otel/metric/noop" +) + +const ( + shadowDroppedMetricName = "nvcf_ai_api_gateway_shadow_requests_dropped" + shadowMetricsScope = "ai-api-gateway-service/gateway" + shadowDroppedModelLabel = "openai_model_name" + shadowDroppedReasonLabel = "reason" +) + +var ( + shadowDroppedCounter otelmetric.Int64Counter = metricnoop.Int64Counter{} + shadowDroppedReasons = []string{ + shadowDroppedReasonBodyReadError, + shadowDroppedReasonBodyRewriteError, + shadowDroppedReasonConcurrencyLimit, + } +) + +func setupShadowMetrics() error { + counter, err := newShadowDroppedCounter(otel.GetMeterProvider().Meter(shadowMetricsScope)) + if err != nil { + return err + } + shadowDroppedCounter = counter + return nil +} + +func newShadowDroppedCounter(meter otelmetric.Meter) (otelmetric.Int64Counter, error) { + counter, err := meter.Int64Counter( + shadowDroppedMetricName, + otelmetric.WithDescription("Number of shadow request dispatches dropped before replay."), + ) + if err != nil { + return nil, fmt.Errorf("create shadow dropped counter: %w", err) + } + return counter, nil +} + +func initializeShadowDropMetrics(targetModels []string) { + for _, targetModel := range targetModels { + for _, reason := range shadowDroppedReasons { + recordShadowDropMetric(context.Background(), targetModel, reason, 0) + } + } +} + +func recordShadowDropMetrics(ctx context.Context, droppedReasons, droppedTargetModels []string) { + for index, reason := range droppedReasons { + if index >= len(droppedTargetModels) { + break + } + recordShadowDropMetric(ctx, droppedTargetModels[index], reason, 1) + } +} + +func recordShadowDropMetric(ctx context.Context, targetModel, reason string, count int64) { + shadowDroppedCounter.Add( + ctx, + count, + otelmetric.WithAttributes( + attribute.String(shadowDroppedModelLabel, targetModel), + attribute.String(shadowDroppedReasonLabel, reason), + ), + ) +} diff --git a/src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics_test.go b/src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics_test.go new file mode 100644 index 000000000..ba359b905 --- /dev/null +++ b/src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics_test.go @@ -0,0 +1,100 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestShadowDroppedMetricInitializesAndCountsPerModel(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + t.Cleanup(func() { + require.NoError(t, provider.Shutdown(context.Background())) + }) + + counter, err := newShadowDroppedCounter(provider.Meter(shadowMetricsScope)) + require.NoError(t, err) + + previousCounter := shadowDroppedCounter + shadowDroppedCounter = counter + t.Cleanup(func() { + shadowDroppedCounter = previousCounter + }) + + initializeShadowDropMetrics([]string{"shadow-a", "shadow-b"}) + recordShadowDispatchSummary( + context.Background(), + []string{"shadow-a", "shadow-b"}, + 0, + 2, + []string{ + shadowDroppedReasonConcurrencyLimit, + shadowDroppedReasonBodyReadError, + }, + []string{"shadow-a", "shadow-b"}, + ) + + require.Equal(t, map[shadowDropMetricLabels]int64{ + {model: "shadow-a", reason: shadowDroppedReasonBodyReadError}: 0, + {model: "shadow-a", reason: shadowDroppedReasonBodyRewriteError}: 0, + {model: "shadow-a", reason: shadowDroppedReasonConcurrencyLimit}: 1, + {model: "shadow-b", reason: shadowDroppedReasonBodyReadError}: 1, + {model: "shadow-b", reason: shadowDroppedReasonBodyRewriteError}: 0, + {model: "shadow-b", reason: shadowDroppedReasonConcurrencyLimit}: 0, + }, collectShadowDroppedCounts(t, reader)) +} + +type shadowDropMetricLabels struct { + model string + reason string +} + +func collectShadowDroppedCounts(t *testing.T, reader *sdkmetric.ManualReader) map[shadowDropMetricLabels]int64 { + t.Helper() + + var resourceMetrics metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &resourceMetrics)) + + counts := make(map[shadowDropMetricLabels]int64) + for _, scopeMetrics := range resourceMetrics.ScopeMetrics { + for _, metric := range scopeMetrics.Metrics { + if metric.Name != shadowDroppedMetricName { + continue + } + sum, ok := metric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + for _, point := range sum.DataPoints { + model, ok := point.Attributes.Value(shadowDroppedModelLabel) + require.True(t, ok) + reason, ok := point.Attributes.Value(shadowDroppedReasonLabel) + require.True(t, ok) + counts[shadowDropMetricLabels{ + model: model.AsString(), + reason: reason.AsString(), + }] = point.Value + } + } + } + return counts +} diff --git a/src/invocation-plane-services/vanity-gateway/go.mod b/src/invocation-plane-services/vanity-gateway/go.mod index 590224345..4109485af 100644 --- a/src/invocation-plane-services/vanity-gateway/go.mod +++ b/src/invocation-plane-services/vanity-gateway/go.mod @@ -20,6 +20,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/exporters/prometheus v0.65.0 + go.opentelemetry.io/otel/metric v1.43.0 go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 @@ -96,7 +97,6 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.65.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect