feat(vanity-gateway): add per-model shadow drop metric

src/invocation-plane-services/vanity-gateway/MODULE.bazel

@@ -101,8 +101,9 @@ use_repo(
     "io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp",
     "io_opentelemetry_go_otel",
     "io_opentelemetry_go_otel_exporters_prometheus",
-    "io_opentelemetry_go_otel_sdk_metric",
+    "io_opentelemetry_go_otel_metric",
     "io_opentelemetry_go_otel_sdk",
+    "io_opentelemetry_go_otel_sdk_metric",
     "io_opentelemetry_go_otel_trace",
     "org_golang_google_grpc",
     "org_golang_x_exp",

src/invocation-plane-services/vanity-gateway/README.md

@@ -369,6 +369,11 @@ curl -v -H 'Host: api.example.com' localhost:10081/health
 curl -v localhost:10083/metrics
 ```
 
+`nvcf_ai_api_gateway_shadow_requests_dropped_total` counts shadow dispatches
+dropped before replay. The `openai_model_name` label identifies the shadow
+target. The `reason` label is one of `body_read_error`, `body_rewrite_error`,
+or `concurrency_limit`.
+
 ## Running a Local OpenAI-compatible Model
 
 Use this only when validating OpenAI-compatible request bodies against a local model server.

src/invocation-plane-services/vanity-gateway/gateway/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "health.go",
         "openai_director.go",
         "shadow.go",
+        "shadow_metrics.go",
         "tracing_attrs.go",
         "vanity_director.go",
         "version.go",
@@ -32,8 +33,11 @@ go_library(
         "@com_github_nvidia_nvcf_go//pkg/nvkit/tracing",
         "@com_github_valyala_bytebufferpool//:bytebufferpool",
         "@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:otelhttp",
+        "@io_opentelemetry_go_otel//:otel",
         "@io_opentelemetry_go_otel//attribute",
         "@io_opentelemetry_go_otel//propagation",
+        "@io_opentelemetry_go_otel_metric//:metric",
+        "@io_opentelemetry_go_otel_metric//noop",
         "@io_opentelemetry_go_otel_trace//:trace",
         "@org_golang_google_grpc//:grpc",
         "@org_golang_x_exp//maps",
@@ -56,6 +60,7 @@ go_test(
         "gateway_test.go",
         "h2_test.go",
         "openai_director_test.go",
+        "shadow_metrics_test.go",
         "shadow_test.go",
         "shadow_tracing_test.go",
         "vanity_director_test.go",
@@ -74,6 +79,8 @@ go_test(
         "@io_opentelemetry_go_otel//codes",
         "@io_opentelemetry_go_otel_sdk//trace",
         "@io_opentelemetry_go_otel_sdk//trace/tracetest",
+        "@io_opentelemetry_go_otel_sdk_metric//:metric",
+        "@io_opentelemetry_go_otel_sdk_metric//metricdata",
         "@io_opentelemetry_go_otel_trace//:trace",
         "@org_uber_go_zap//:zap",
         "@org_uber_go_zap//zaptest/observer",

src/invocation-plane-services/vanity-gateway/gateway/gateway.go

@@ -104,6 +104,9 @@ func NewNVCFGateway(logger *logs.ZapLogger, config Config) (*NVCFGateway, error)
 	if err := middleware.SetupHTTPMetrics(); err != nil {
 		return nil, fmt.Errorf("failed to setup HTTP metrics: %w", err)
 	}
+	if err := setupShadowMetrics(); err != nil {
+		return nil, fmt.Errorf("failed to setup shadow metrics: %w", err)
+	}
 
 	mappings, err := gatewayConfig.SetupConfigWithConfigPath(config.MappingPath)
 	if err != nil {

src/invocation-plane-services/vanity-gateway/gateway/openai_director.go

@@ -236,6 +236,7 @@ func buildModelMapping(
 		if entry.OutgoingPathOverride != "" {
 			pathOverride = &entry.OutgoingPathOverride
 		}
+		initializeShadowDropMetrics(entry.ShadowModelNames)
 		modelNameToNVCFUrl[modelName] = FunctionInfo{
 			functionId:                     entry.FunctionId,
 			functionVersionId:              entry.FunctionVersionId,

src/invocation-plane-services/vanity-gateway/gateway/shadow.go

@@ -123,6 +123,8 @@ func shadowContext(req *http.Request, cancelOnClientDisconnect bool) (context.Co
 }
 
 func recordShadowDispatchSummary(ctx context.Context, targetModels []string, dispatchedCount int, droppedCount int, droppedReasons []string, droppedTargetModels []string) {
+	recordShadowDropMetrics(ctx, droppedReasons, droppedTargetModels)
+
 	span := trace.SpanFromContext(ctx)
 	attrs := []attribute.KeyValue{
 		traceAttrShadowDispatched.Bool(dispatchedCount > 0),

src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics.go

@@ -0,0 +1,92 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gateway
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	otelmetric "go.opentelemetry.io/otel/metric"
+	metricnoop "go.opentelemetry.io/otel/metric/noop"
+)
+
+const (
+	shadowDroppedMetricName  = "nvcf_ai_api_gateway_shadow_requests_dropped"
+	shadowMetricsScope       = "ai-api-gateway-service/gateway"
+	shadowDroppedModelLabel  = "openai_model_name"
+	shadowDroppedReasonLabel = "reason"
+)
+
+var (
+	shadowDroppedCounter otelmetric.Int64Counter = metricnoop.Int64Counter{}
+	shadowDroppedReasons                         = []string{
+		shadowDroppedReasonBodyReadError,
+		shadowDroppedReasonBodyRewriteError,
+		shadowDroppedReasonConcurrencyLimit,
+	}
+)
+
+func setupShadowMetrics() error {
+	counter, err := newShadowDroppedCounter(otel.GetMeterProvider().Meter(shadowMetricsScope))
+	if err != nil {
+		return err
+	}
+	shadowDroppedCounter = counter
+	return nil
+}
+
+func newShadowDroppedCounter(meter otelmetric.Meter) (otelmetric.Int64Counter, error) {
+	counter, err := meter.Int64Counter(
+		shadowDroppedMetricName,
+		otelmetric.WithDescription("Number of shadow request dispatches dropped before replay."),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("create shadow dropped counter: %w", err)
+	}
+	return counter, nil
+}
+
+func initializeShadowDropMetrics(targetModels []string) {
+	for _, targetModel := range targetModels {
+		for _, reason := range shadowDroppedReasons {
+			recordShadowDropMetric(context.Background(), targetModel, reason, 0)
+		}
+	}
+}
+
+func recordShadowDropMetrics(ctx context.Context, droppedReasons, droppedTargetModels []string) {
+	for index, reason := range droppedReasons {
+		if index >= len(droppedTargetModels) {
+			break
+		}
+		recordShadowDropMetric(ctx, droppedTargetModels[index], reason, 1)
+	}
+}
+
+func recordShadowDropMetric(ctx context.Context, targetModel, reason string, count int64) {
+	shadowDroppedCounter.Add(
+		ctx,
+		count,
+		otelmetric.WithAttributes(
+			attribute.String(shadowDroppedModelLabel, targetModel),
+			attribute.String(shadowDroppedReasonLabel, reason),
+		),
+	)
+}

src/invocation-plane-services/vanity-gateway/gateway/shadow_metrics_test.go

@@ -0,0 +1,100 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gateway
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+)
+
+func TestShadowDroppedMetricInitializesAndCountsPerModel(t *testing.T) {
+	reader := sdkmetric.NewManualReader()
+	provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
+	t.Cleanup(func() {
+		require.NoError(t, provider.Shutdown(context.Background()))
+	})
+
+	counter, err := newShadowDroppedCounter(provider.Meter(shadowMetricsScope))
+	require.NoError(t, err)
+
+	previousCounter := shadowDroppedCounter
+	shadowDroppedCounter = counter
+	t.Cleanup(func() {
+		shadowDroppedCounter = previousCounter
+	})
+
+	initializeShadowDropMetrics([]string{"shadow-a", "shadow-b"})
+	recordShadowDispatchSummary(
+		context.Background(),
+		[]string{"shadow-a", "shadow-b"},
+		0,
+		2,
+		[]string{
+			shadowDroppedReasonConcurrencyLimit,
+			shadowDroppedReasonBodyReadError,
+		},
+		[]string{"shadow-a", "shadow-b"},
+	)
+
+	require.Equal(t, map[shadowDropMetricLabels]int64{
+		{model: "shadow-a", reason: shadowDroppedReasonBodyReadError}:    0,
+		{model: "shadow-a", reason: shadowDroppedReasonBodyRewriteError}: 0,
+		{model: "shadow-a", reason: shadowDroppedReasonConcurrencyLimit}: 1,
+		{model: "shadow-b", reason: shadowDroppedReasonBodyReadError}:    1,
+		{model: "shadow-b", reason: shadowDroppedReasonBodyRewriteError}: 0,
+		{model: "shadow-b", reason: shadowDroppedReasonConcurrencyLimit}: 0,
+	}, collectShadowDroppedCounts(t, reader))
+}
+
+type shadowDropMetricLabels struct {
+	model  string
+	reason string
+}
+
+func collectShadowDroppedCounts(t *testing.T, reader *sdkmetric.ManualReader) map[shadowDropMetricLabels]int64 {
+	t.Helper()
+
+	var resourceMetrics metricdata.ResourceMetrics
+	require.NoError(t, reader.Collect(context.Background(), &resourceMetrics))
+
+	counts := make(map[shadowDropMetricLabels]int64)
+	for _, scopeMetrics := range resourceMetrics.ScopeMetrics {
+		for _, metric := range scopeMetrics.Metrics {
+			if metric.Name != shadowDroppedMetricName {
+				continue
+			}
+			sum, ok := metric.Data.(metricdata.Sum[int64])
+			require.True(t, ok)
+			for _, point := range sum.DataPoints {
+				model, ok := point.Attributes.Value(shadowDroppedModelLabel)
+				require.True(t, ok)
+				reason, ok := point.Attributes.Value(shadowDroppedReasonLabel)
+				require.True(t, ok)
+				counts[shadowDropMetricLabels{
+					model:  model.AsString(),
+					reason: reason.AsString(),
+				}] = point.Value
+			}
+		}
+	}
+	return counts
+}

src/invocation-plane-services/vanity-gateway/go.mod

@@ -20,6 +20,7 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0
 	go.opentelemetry.io/otel v1.43.0
 	go.opentelemetry.io/otel/exporters/prometheus v0.65.0
+	go.opentelemetry.io/otel/metric v1.43.0
 	go.opentelemetry.io/otel/sdk v1.43.0
 	go.opentelemetry.io/otel/sdk/metric v1.43.0
 	go.opentelemetry.io/otel/trace v1.43.0
@@ -96,7 +97,6 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.65.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
-	go.opentelemetry.io/otel/metric v1.43.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/automaxprocs v1.6.0 // indirect