From 085719f606ea6cdfb971323bb57da3dd97628e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20St=C3=A4bler?= Date: Wed, 20 May 2026 09:19:35 +0200 Subject: [PATCH 1/6] Add config option to disable high-cardinality metric attributes Add a `metrics.high-cardinality.disable` key to the config-observability ConfigMap that allows stripping unbounded metric attributes (cloudevents.type, messaging.destination.name) via an OTel View. This helps prevent OOM issues caused by high metric cardinality in production without changing any instrumentation code. --- config/core/configmaps/observability.yaml | 5 +++ pkg/observability/config.go | 16 ++++++++- pkg/observability/config_test.go | 15 ++++++-- pkg/observability/otel/otel.go | 21 ++++++++++- pkg/observability/otel/otel_test.go | 43 +++++++++++++++++++++++ 5 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 pkg/observability/otel/otel_test.go diff --git a/config/core/configmaps/observability.yaml b/config/core/configmaps/observability.yaml index 41f3bee1fd6..5a4b077967c 100644 --- a/config/core/configmaps/observability.yaml +++ b/config/core/configmaps/observability.yaml @@ -58,6 +58,11 @@ data: # If a zero or negative value is passed the default reporting OTel period is used (60 secs). metrics-export-interval: 60s + # metrics.high-cardinality.disable disables high-cardinality metric attributes such as + # cloudevents.type and messaging.destination.name. Enabling this can help prevent + # OOM issues caused by unbounded metric cardinality in production. + metrics.high-cardinality.disable: "false" + # sink-event-error-reporting.enable whether the adapter reports a kube event to the CRD indicating # a failure to send a cloud event to the sink. sink-event-error-reporting.enable: "false" diff --git a/pkg/observability/config.go b/pkg/observability/config.go index 9f0673caf06..10be422c8ee 100644 --- a/pkg/observability/config.go +++ b/pkg/observability/config.go @@ -32,6 +32,12 @@ const ( // DefaultEnableSinkEventErrorReporting is used to set the default sink event error reporting value DefaultEnableSinkEventErrorReporting = false + // DisableHighCardinalityMetricsKey is the CM key to disable high-cardinality metric attributes + DisableHighCardinalityMetricsKey = "metrics.high-cardinality.disable" + + // DefaultDisableHighCardinalityMetrics is the default value for disabling high-cardinality metrics + DefaultDisableHighCardinalityMetrics = false + // DefaultMetricsPort is the default port used for prometheus metrics if the prometheus protocol is used DefaultMetricsPort = 9092 ) @@ -50,12 +56,17 @@ type Config struct { // EnableSinkEventErrorReporting specifies whether we should emit a k8s // event when delivery to a sink fails EnableSinkEventErrorReporting bool `json:"enableSinkEventErrorReporting"` + + // DisableHighCardinalityMetrics specifies whether high-cardinality attributes + // (e.g. cloudevents.type, messaging.destination.name) are stripped from metrics + DisableHighCardinalityMetrics bool `json:"disableHighCardinalityMetrics"` } func DefaultConfig() *Config { return &Config{ BaseConfig: *pkgo11y.DefaultConfig(), EnableSinkEventErrorReporting: DefaultEnableSinkEventErrorReporting, + DisableHighCardinalityMetrics: DefaultDisableHighCardinalityMetrics, } } @@ -74,7 +85,10 @@ func NewFromMap(m map[string]string) (*Config, error) { c.BaseConfig.Metrics.Endpoint = fmt.Sprintf(":%d", DefaultMetricsPort) } - err := configmap.Parse(m, configmap.As(EnableSinkEventErrorReportingKey, &c.EnableSinkEventErrorReporting)) + err := configmap.Parse(m, + configmap.As(EnableSinkEventErrorReportingKey, &c.EnableSinkEventErrorReporting), + configmap.As(DisableHighCardinalityMetricsKey, &c.DisableHighCardinalityMetrics), + ) if err != nil { fmt.Printf("failed to parse enable-sink-error-reporting: %s\n", err.Error()) return c, err diff --git a/pkg/observability/config_test.go b/pkg/observability/config_test.go index b67d5e14eed..4bf64a663d0 100644 --- a/pkg/observability/config_test.go +++ b/pkg/observability/config_test.go @@ -23,8 +23,11 @@ import ( ) func TestNewFromMap(t *testing.T) { - configWithOverride := DefaultConfig() - configWithOverride.EnableSinkEventErrorReporting = true + configWithSinkEventErrorReporting := DefaultConfig() + configWithSinkEventErrorReporting.EnableSinkEventErrorReporting = true + + configWithHighCardinalityDisabled := DefaultConfig() + configWithHighCardinalityDisabled.DisableHighCardinalityMetrics = true testCases := map[string]struct { m map[string]string @@ -39,7 +42,13 @@ func TestNewFromMap(t *testing.T) { m: map[string]string{ EnableSinkEventErrorReportingKey: "true", }, - want: configWithOverride, + want: configWithSinkEventErrorReporting, + }, + "disable high cardinality metrics": { + m: map[string]string{ + DisableHighCardinalityMetricsKey: "true", + }, + want: configWithHighCardinalityDisabled, }, "valid keys, invalid sink event error reporting value": { m: map[string]string{ diff --git a/pkg/observability/otel/otel.go b/pkg/observability/otel/otel.go index cfcdc684d6e..5a62fcc973b 100644 --- a/pkg/observability/otel/otel.go +++ b/pkg/observability/otel/otel.go @@ -19,8 +19,10 @@ package otel import ( "context" + ceo11y "github.com/cloudevents/sdk-go/v2/observability" "go.opentelemetry.io/contrib/instrumentation/runtime" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/sdk/metric" sdkresource "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" @@ -60,10 +62,15 @@ func SetupObservabilityOrDie( otelResource := resource.Default(component) + meterOpts := []metric.Option{metric.WithResource(otelResource)} + if cfg.DisableHighCardinalityMetrics { + meterOpts = append(meterOpts, metric.WithView(highCardinalityFilter())) + } + meterProvider, err := metrics.NewMeterProvider( ctx, cfg.Metrics, - metric.WithResource(otelResource), + meterOpts..., ) if err != nil { logger.Fatalw("failed to set up meter provider", zap.Error(err)) @@ -155,3 +162,15 @@ func GetObservabilityConfig(ctx context.Context) (*observability.Config, error) return configmap.Parse(cm) } + +func highCardinalityFilter() metric.View { + return metric.NewView( + metric.Instrument{Name: "kn.eventing.*"}, + metric.Stream{ + AttributeFilter: attribute.NewDenyKeysFilter( + attribute.Key(ceo11y.TypeAttr), // "cloudevents.type" + attribute.Key(observability.MessagingDestinationName), // "messaging.destination.name" + ), + }, + ) +} diff --git a/pkg/observability/otel/otel_test.go b/pkg/observability/otel/otel_test.go new file mode 100644 index 00000000000..dccb827b303 --- /dev/null +++ b/pkg/observability/otel/otel_test.go @@ -0,0 +1,43 @@ +package otel + +import ( + "testing" + + ceo11y "github.com/cloudevents/sdk-go/v2/observability" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + + "knative.dev/eventing/pkg/observability" +) + +func TestHighCardinalityFilter(t *testing.T) { + view := highCardinalityFilter() + + stream, ok := view(metric.Instrument{Name: "kn.eventing.dispatch.duration"}) + assert.True(t, ok, "view should match kn.eventing.* instruments") + assert.NotNil(t, stream.AttributeFilter) + + denied := []attribute.KeyValue{ + attribute.String(ceo11y.TypeAttr, "com.example.event"), + attribute.String(string(observability.MessagingDestinationName), "my-destination"), + } + for _, kv := range denied { + assert.False(t, stream.AttributeFilter(kv), "attribute %s should be denied", kv.Key) + } + + allowed := []attribute.KeyValue{ + attribute.String("messaging.system", "knative"), + attribute.Int("http.response.status_code", 200), + } + for _, kv := range allowed { + assert.True(t, stream.AttributeFilter(kv), "attribute %s should be allowed", kv.Key) + } +} + +func TestHighCardinalityFilterDoesNotMatchOtherInstruments(t *testing.T) { + view := highCardinalityFilter() + + _, ok := view(metric.Instrument{Name: "http.server.request.duration"}) + assert.False(t, ok, "view should not match non kn.eventing.* instruments") +} From 2b4ff1c80b8bd82fa393cec82ae2b14e95043bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20St=C3=A4bler?= Date: Wed, 20 May 2026 10:03:18 +0200 Subject: [PATCH 2/6] Fix CI: add license boilerplate and update example checksum --- config/core/configmaps/observability.yaml | 2 +- pkg/observability/otel/otel_test.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/config/core/configmaps/observability.yaml b/config/core/configmaps/observability.yaml index 5a4b077967c..00cec8ecd3a 100644 --- a/config/core/configmaps/observability.yaml +++ b/config/core/configmaps/observability.yaml @@ -23,7 +23,7 @@ metadata: app.kubernetes.io/version: devel app.kubernetes.io/name: knative-eventing annotations: - knative.dev/example-checksum: "0270bb17" + knative.dev/example-checksum: "1fdd6513" data: _example: | ################################ diff --git a/pkg/observability/otel/otel_test.go b/pkg/observability/otel/otel_test.go index dccb827b303..037356c47b9 100644 --- a/pkg/observability/otel/otel_test.go +++ b/pkg/observability/otel/otel_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package otel import ( From 4b6d08191932de2865c44ce5ebe15cc0692b3977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20St=C3=A4bler?= Date: Wed, 20 May 2026 11:50:08 +0200 Subject: [PATCH 3/6] Replace boolean toggle with configurable metric attributes deny list Replace DisableHighCardinalityMetrics boolean with a configurable metrics.attributes.deny comma-separated list in config-observability. Users can specify exactly which metric attribute keys to strip from kn.eventing.* metrics, allowing fine-grained control over cardinality. Default is empty (no filtering), preserving existing behavior. --- config/core/configmaps/observability.yaml | 10 ++++----- pkg/observability/config.go | 26 ++++++++++++++--------- pkg/observability/config_test.go | 10 ++++----- pkg/observability/otel/otel.go | 16 +++++++------- pkg/observability/otel/otel_test.go | 15 ++++++------- 5 files changed, 40 insertions(+), 37 deletions(-) diff --git a/config/core/configmaps/observability.yaml b/config/core/configmaps/observability.yaml index 00cec8ecd3a..336ccba1f9e 100644 --- a/config/core/configmaps/observability.yaml +++ b/config/core/configmaps/observability.yaml @@ -23,7 +23,7 @@ metadata: app.kubernetes.io/version: devel app.kubernetes.io/name: knative-eventing annotations: - knative.dev/example-checksum: "1fdd6513" + knative.dev/example-checksum: "6010dfc0" data: _example: | ################################ @@ -58,10 +58,10 @@ data: # If a zero or negative value is passed the default reporting OTel period is used (60 secs). metrics-export-interval: 60s - # metrics.high-cardinality.disable disables high-cardinality metric attributes such as - # cloudevents.type and messaging.destination.name. Enabling this can help prevent - # OOM issues caused by unbounded metric cardinality in production. - metrics.high-cardinality.disable: "false" + # metrics.attributes.deny is a comma-separated list of metric attribute keys to filter + # out from kn.eventing.* metrics. This can help prevent OOM issues caused by unbounded + # metric cardinality in production (e.g. cloudevents.type, messaging.destination.name). + metrics.attributes.deny: "" # sink-event-error-reporting.enable whether the adapter reports a kube event to the CRD indicating # a failure to send a cloud event to the sink. diff --git a/pkg/observability/config.go b/pkg/observability/config.go index 10be422c8ee..6d94b21068e 100644 --- a/pkg/observability/config.go +++ b/pkg/observability/config.go @@ -19,6 +19,7 @@ package observability import ( "context" "fmt" + "strings" configmap "knative.dev/pkg/configmap/parser" pkgo11y "knative.dev/pkg/observability" @@ -32,11 +33,8 @@ const ( // DefaultEnableSinkEventErrorReporting is used to set the default sink event error reporting value DefaultEnableSinkEventErrorReporting = false - // DisableHighCardinalityMetricsKey is the CM key to disable high-cardinality metric attributes - DisableHighCardinalityMetricsKey = "metrics.high-cardinality.disable" - - // DefaultDisableHighCardinalityMetrics is the default value for disabling high-cardinality metrics - DefaultDisableHighCardinalityMetrics = false + // MetricAttributesDenyListKey is the CM key for a comma-separated list of metric attribute keys to filter out + MetricAttributesDenyListKey = "metrics.attributes.deny" // DefaultMetricsPort is the default port used for prometheus metrics if the prometheus protocol is used DefaultMetricsPort = 9092 @@ -57,16 +55,15 @@ type Config struct { // event when delivery to a sink fails EnableSinkEventErrorReporting bool `json:"enableSinkEventErrorReporting"` - // DisableHighCardinalityMetrics specifies whether high-cardinality attributes - // (e.g. cloudevents.type, messaging.destination.name) are stripped from metrics - DisableHighCardinalityMetrics bool `json:"disableHighCardinalityMetrics"` + // MetricAttributesDenyList is a list of metric attribute keys to filter out + // from kn.eventing.* metrics (e.g. cloudevents.type, messaging.destination.name) + MetricAttributesDenyList []string `json:"metricAttributesDenyList,omitempty"` } func DefaultConfig() *Config { return &Config{ BaseConfig: *pkgo11y.DefaultConfig(), EnableSinkEventErrorReporting: DefaultEnableSinkEventErrorReporting, - DisableHighCardinalityMetrics: DefaultDisableHighCardinalityMetrics, } } @@ -85,9 +82,18 @@ func NewFromMap(m map[string]string) (*Config, error) { c.BaseConfig.Metrics.Endpoint = fmt.Sprintf(":%d", DefaultMetricsPort) } + if v, ok := m[MetricAttributesDenyListKey]; ok && v != "" { + parts := strings.Split(v, ",") + c.MetricAttributesDenyList = make([]string, 0, len(parts)) + for _, p := range parts { + if t := strings.TrimSpace(p); t != "" { + c.MetricAttributesDenyList = append(c.MetricAttributesDenyList, t) + } + } + } + err := configmap.Parse(m, configmap.As(EnableSinkEventErrorReportingKey, &c.EnableSinkEventErrorReporting), - configmap.As(DisableHighCardinalityMetricsKey, &c.DisableHighCardinalityMetrics), ) if err != nil { fmt.Printf("failed to parse enable-sink-error-reporting: %s\n", err.Error()) diff --git a/pkg/observability/config_test.go b/pkg/observability/config_test.go index 4bf64a663d0..1aba5cfc994 100644 --- a/pkg/observability/config_test.go +++ b/pkg/observability/config_test.go @@ -26,8 +26,8 @@ func TestNewFromMap(t *testing.T) { configWithSinkEventErrorReporting := DefaultConfig() configWithSinkEventErrorReporting.EnableSinkEventErrorReporting = true - configWithHighCardinalityDisabled := DefaultConfig() - configWithHighCardinalityDisabled.DisableHighCardinalityMetrics = true + configWithDenyList := DefaultConfig() + configWithDenyList.MetricAttributesDenyList = []string{"cloudevents.type", "messaging.destination.name"} testCases := map[string]struct { m map[string]string @@ -44,11 +44,11 @@ func TestNewFromMap(t *testing.T) { }, want: configWithSinkEventErrorReporting, }, - "disable high cardinality metrics": { + "metric attributes deny list": { m: map[string]string{ - DisableHighCardinalityMetricsKey: "true", + MetricAttributesDenyListKey: "cloudevents.type, messaging.destination.name", }, - want: configWithHighCardinalityDisabled, + want: configWithDenyList, }, "valid keys, invalid sink event error reporting value": { m: map[string]string{ diff --git a/pkg/observability/otel/otel.go b/pkg/observability/otel/otel.go index 5a62fcc973b..d494bf9696b 100644 --- a/pkg/observability/otel/otel.go +++ b/pkg/observability/otel/otel.go @@ -19,7 +19,6 @@ package otel import ( "context" - ceo11y "github.com/cloudevents/sdk-go/v2/observability" "go.opentelemetry.io/contrib/instrumentation/runtime" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -63,8 +62,8 @@ func SetupObservabilityOrDie( otelResource := resource.Default(component) meterOpts := []metric.Option{metric.WithResource(otelResource)} - if cfg.DisableHighCardinalityMetrics { - meterOpts = append(meterOpts, metric.WithView(highCardinalityFilter())) + if len(cfg.MetricAttributesDenyList) > 0 { + meterOpts = append(meterOpts, metric.WithView(metricAttributesDenyFilter(cfg.MetricAttributesDenyList))) } meterProvider, err := metrics.NewMeterProvider( @@ -163,14 +162,15 @@ func GetObservabilityConfig(ctx context.Context) (*observability.Config, error) return configmap.Parse(cm) } -func highCardinalityFilter() metric.View { +func metricAttributesDenyFilter(denyList []string) metric.View { + keys := make([]attribute.Key, len(denyList)) + for i, k := range denyList { + keys[i] = attribute.Key(k) + } return metric.NewView( metric.Instrument{Name: "kn.eventing.*"}, metric.Stream{ - AttributeFilter: attribute.NewDenyKeysFilter( - attribute.Key(ceo11y.TypeAttr), // "cloudevents.type" - attribute.Key(observability.MessagingDestinationName), // "messaging.destination.name" - ), + AttributeFilter: attribute.NewDenyKeysFilter(keys...), }, ) } diff --git a/pkg/observability/otel/otel_test.go b/pkg/observability/otel/otel_test.go index 037356c47b9..ebf8fa0189a 100644 --- a/pkg/observability/otel/otel_test.go +++ b/pkg/observability/otel/otel_test.go @@ -19,24 +19,21 @@ package otel import ( "testing" - ceo11y "github.com/cloudevents/sdk-go/v2/observability" "github.com/stretchr/testify/assert" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/sdk/metric" - - "knative.dev/eventing/pkg/observability" ) -func TestHighCardinalityFilter(t *testing.T) { - view := highCardinalityFilter() +func TestMetricAttributesDenyFilter(t *testing.T) { + view := metricAttributesDenyFilter([]string{"cloudevents.type", "messaging.destination.name"}) stream, ok := view(metric.Instrument{Name: "kn.eventing.dispatch.duration"}) assert.True(t, ok, "view should match kn.eventing.* instruments") assert.NotNil(t, stream.AttributeFilter) denied := []attribute.KeyValue{ - attribute.String(ceo11y.TypeAttr, "com.example.event"), - attribute.String(string(observability.MessagingDestinationName), "my-destination"), + attribute.String("cloudevents.type", "com.example.event"), + attribute.String("messaging.destination.name", "my-destination"), } for _, kv := range denied { assert.False(t, stream.AttributeFilter(kv), "attribute %s should be denied", kv.Key) @@ -51,8 +48,8 @@ func TestHighCardinalityFilter(t *testing.T) { } } -func TestHighCardinalityFilterDoesNotMatchOtherInstruments(t *testing.T) { - view := highCardinalityFilter() +func TestMetricAttributesDenyFilterDoesNotMatchOtherInstruments(t *testing.T) { + view := metricAttributesDenyFilter([]string{"cloudevents.type"}) _, ok := view(metric.Instrument{Name: "http.server.request.duration"}) assert.False(t, ok, "view should not match non kn.eventing.* instruments") From ca71a7abef1c0dcd7a995e42933f01bee7a9d213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20St=C3=A4bler?= Date: Wed, 20 May 2026 15:40:01 +0200 Subject: [PATCH 4/6] Apply metric attributes deny filter to all instruments The deny filter was scoped to kn.eventing.* instruments only, but high-cardinality attributes like cloudevents.type also appear on http.* metrics via the otelhttp instrumentation. Apply the filter to all instruments so it covers every metric that inherits labels from the context labeler. --- config/core/configmaps/observability.yaml | 4 ++-- pkg/observability/otel/otel.go | 2 +- pkg/observability/otel/otel_test.go | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/config/core/configmaps/observability.yaml b/config/core/configmaps/observability.yaml index 336ccba1f9e..b50b10aea1e 100644 --- a/config/core/configmaps/observability.yaml +++ b/config/core/configmaps/observability.yaml @@ -23,7 +23,7 @@ metadata: app.kubernetes.io/version: devel app.kubernetes.io/name: knative-eventing annotations: - knative.dev/example-checksum: "6010dfc0" + knative.dev/example-checksum: "afa5507a" data: _example: | ################################ @@ -59,7 +59,7 @@ data: metrics-export-interval: 60s # metrics.attributes.deny is a comma-separated list of metric attribute keys to filter - # out from kn.eventing.* metrics. This can help prevent OOM issues caused by unbounded + # out from all metrics. This can help prevent OOM issues caused by unbounded # metric cardinality in production (e.g. cloudevents.type, messaging.destination.name). metrics.attributes.deny: "" diff --git a/pkg/observability/otel/otel.go b/pkg/observability/otel/otel.go index d494bf9696b..0ede3a5d579 100644 --- a/pkg/observability/otel/otel.go +++ b/pkg/observability/otel/otel.go @@ -168,7 +168,7 @@ func metricAttributesDenyFilter(denyList []string) metric.View { keys[i] = attribute.Key(k) } return metric.NewView( - metric.Instrument{Name: "kn.eventing.*"}, + metric.Instrument{Name: "*"}, metric.Stream{ AttributeFilter: attribute.NewDenyKeysFilter(keys...), }, diff --git a/pkg/observability/otel/otel_test.go b/pkg/observability/otel/otel_test.go index ebf8fa0189a..561e8af34da 100644 --- a/pkg/observability/otel/otel_test.go +++ b/pkg/observability/otel/otel_test.go @@ -48,9 +48,10 @@ func TestMetricAttributesDenyFilter(t *testing.T) { } } -func TestMetricAttributesDenyFilterDoesNotMatchOtherInstruments(t *testing.T) { +func TestMetricAttributesDenyFilterMatchesAllInstruments(t *testing.T) { view := metricAttributesDenyFilter([]string{"cloudevents.type"}) - _, ok := view(metric.Instrument{Name: "http.server.request.duration"}) - assert.False(t, ok, "view should not match non kn.eventing.* instruments") + stream, ok := view(metric.Instrument{Name: "http.server.request.duration"}) + assert.True(t, ok, "view should match all instruments") + assert.NotNil(t, stream.AttributeFilter) } From 99cabfbb6b4e1f10d1bc94139dc2e162e0175866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20St=C3=A4bler?= Date: Thu, 21 May 2026 08:58:55 +0200 Subject: [PATCH 5/6] Update comment: deny list applies to all metrics, not just kn.eventing.* --- pkg/observability/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/observability/config.go b/pkg/observability/config.go index 6d94b21068e..5333461b2e6 100644 --- a/pkg/observability/config.go +++ b/pkg/observability/config.go @@ -56,7 +56,7 @@ type Config struct { EnableSinkEventErrorReporting bool `json:"enableSinkEventErrorReporting"` // MetricAttributesDenyList is a list of metric attribute keys to filter out - // from kn.eventing.* metrics (e.g. cloudevents.type, messaging.destination.name) + // from all metrics (e.g. cloudevents.type, messaging.destination.name) MetricAttributesDenyList []string `json:"metricAttributesDenyList,omitempty"` } From c288ebb0d6a57bb0da28ff7070a5779fc1440218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20St=C3=A4bler?= Date: Thu, 21 May 2026 17:45:05 +0200 Subject: [PATCH 6/6] Use knative.dev/pkg for metric attributes deny list Remove eventing-specific MetricAttributesDenyList field, parsing, and filter function. Use cfg.Metrics.AttributesDenyList() and metrics.MetricAttributesDenyFilter from knative.dev/pkg instead. Rename ConfigMap key from metrics.attributes.deny to metrics-attributes-deny to match knative.dev/pkg convention. Depends on: https://github.com/knative/pkg/pull/3356 --- config/core/configmaps/observability.yaml | 4 ++-- pkg/observability/config.go | 18 ------------------ pkg/observability/config_test.go | 4 ++-- pkg/observability/otel/otel.go | 18 ++---------------- pkg/observability/otel/otel_test.go | 6 ++++-- 5 files changed, 10 insertions(+), 40 deletions(-) diff --git a/config/core/configmaps/observability.yaml b/config/core/configmaps/observability.yaml index b50b10aea1e..417bf4bb788 100644 --- a/config/core/configmaps/observability.yaml +++ b/config/core/configmaps/observability.yaml @@ -58,10 +58,10 @@ data: # If a zero or negative value is passed the default reporting OTel period is used (60 secs). metrics-export-interval: 60s - # metrics.attributes.deny is a comma-separated list of metric attribute keys to filter + # metrics-attributes-deny is a comma-separated list of metric attribute keys to filter # out from all metrics. This can help prevent OOM issues caused by unbounded # metric cardinality in production (e.g. cloudevents.type, messaging.destination.name). - metrics.attributes.deny: "" + metrics-attributes-deny: "" # sink-event-error-reporting.enable whether the adapter reports a kube event to the CRD indicating # a failure to send a cloud event to the sink. diff --git a/pkg/observability/config.go b/pkg/observability/config.go index 5333461b2e6..ce1dcad1581 100644 --- a/pkg/observability/config.go +++ b/pkg/observability/config.go @@ -19,7 +19,6 @@ package observability import ( "context" "fmt" - "strings" configmap "knative.dev/pkg/configmap/parser" pkgo11y "knative.dev/pkg/observability" @@ -33,9 +32,6 @@ const ( // DefaultEnableSinkEventErrorReporting is used to set the default sink event error reporting value DefaultEnableSinkEventErrorReporting = false - // MetricAttributesDenyListKey is the CM key for a comma-separated list of metric attribute keys to filter out - MetricAttributesDenyListKey = "metrics.attributes.deny" - // DefaultMetricsPort is the default port used for prometheus metrics if the prometheus protocol is used DefaultMetricsPort = 9092 ) @@ -54,10 +50,6 @@ type Config struct { // EnableSinkEventErrorReporting specifies whether we should emit a k8s // event when delivery to a sink fails EnableSinkEventErrorReporting bool `json:"enableSinkEventErrorReporting"` - - // MetricAttributesDenyList is a list of metric attribute keys to filter out - // from all metrics (e.g. cloudevents.type, messaging.destination.name) - MetricAttributesDenyList []string `json:"metricAttributesDenyList,omitempty"` } func DefaultConfig() *Config { @@ -82,16 +74,6 @@ func NewFromMap(m map[string]string) (*Config, error) { c.BaseConfig.Metrics.Endpoint = fmt.Sprintf(":%d", DefaultMetricsPort) } - if v, ok := m[MetricAttributesDenyListKey]; ok && v != "" { - parts := strings.Split(v, ",") - c.MetricAttributesDenyList = make([]string, 0, len(parts)) - for _, p := range parts { - if t := strings.TrimSpace(p); t != "" { - c.MetricAttributesDenyList = append(c.MetricAttributesDenyList, t) - } - } - } - err := configmap.Parse(m, configmap.As(EnableSinkEventErrorReportingKey, &c.EnableSinkEventErrorReporting), ) diff --git a/pkg/observability/config_test.go b/pkg/observability/config_test.go index 1aba5cfc994..3b4040c56f8 100644 --- a/pkg/observability/config_test.go +++ b/pkg/observability/config_test.go @@ -27,7 +27,7 @@ func TestNewFromMap(t *testing.T) { configWithSinkEventErrorReporting.EnableSinkEventErrorReporting = true configWithDenyList := DefaultConfig() - configWithDenyList.MetricAttributesDenyList = []string{"cloudevents.type", "messaging.destination.name"} + configWithDenyList.Metrics.AttributesDeny = "cloudevents.type, messaging.destination.name" testCases := map[string]struct { m map[string]string @@ -46,7 +46,7 @@ func TestNewFromMap(t *testing.T) { }, "metric attributes deny list": { m: map[string]string{ - MetricAttributesDenyListKey: "cloudevents.type, messaging.destination.name", + "metrics-attributes-deny": "cloudevents.type, messaging.destination.name", }, want: configWithDenyList, }, diff --git a/pkg/observability/otel/otel.go b/pkg/observability/otel/otel.go index 0ede3a5d579..01fbc66a725 100644 --- a/pkg/observability/otel/otel.go +++ b/pkg/observability/otel/otel.go @@ -21,7 +21,6 @@ import ( "go.opentelemetry.io/contrib/instrumentation/runtime" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/sdk/metric" sdkresource "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" @@ -62,8 +61,8 @@ func SetupObservabilityOrDie( otelResource := resource.Default(component) meterOpts := []metric.Option{metric.WithResource(otelResource)} - if len(cfg.MetricAttributesDenyList) > 0 { - meterOpts = append(meterOpts, metric.WithView(metricAttributesDenyFilter(cfg.MetricAttributesDenyList))) + if denyList := cfg.Metrics.AttributesDenyList(); len(denyList) > 0 { + meterOpts = append(meterOpts, metric.WithView(metrics.MetricAttributesDenyFilter(denyList))) } meterProvider, err := metrics.NewMeterProvider( @@ -161,16 +160,3 @@ func GetObservabilityConfig(ctx context.Context) (*observability.Config, error) return configmap.Parse(cm) } - -func metricAttributesDenyFilter(denyList []string) metric.View { - keys := make([]attribute.Key, len(denyList)) - for i, k := range denyList { - keys[i] = attribute.Key(k) - } - return metric.NewView( - metric.Instrument{Name: "*"}, - metric.Stream{ - AttributeFilter: attribute.NewDenyKeysFilter(keys...), - }, - ) -} diff --git a/pkg/observability/otel/otel_test.go b/pkg/observability/otel/otel_test.go index 561e8af34da..95b26555d90 100644 --- a/pkg/observability/otel/otel_test.go +++ b/pkg/observability/otel/otel_test.go @@ -22,10 +22,12 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/sdk/metric" + + "knative.dev/pkg/observability/metrics" ) func TestMetricAttributesDenyFilter(t *testing.T) { - view := metricAttributesDenyFilter([]string{"cloudevents.type", "messaging.destination.name"}) + view := metrics.MetricAttributesDenyFilter([]string{"cloudevents.type", "messaging.destination.name"}) stream, ok := view(metric.Instrument{Name: "kn.eventing.dispatch.duration"}) assert.True(t, ok, "view should match kn.eventing.* instruments") @@ -49,7 +51,7 @@ func TestMetricAttributesDenyFilter(t *testing.T) { } func TestMetricAttributesDenyFilterMatchesAllInstruments(t *testing.T) { - view := metricAttributesDenyFilter([]string{"cloudevents.type"}) + view := metrics.MetricAttributesDenyFilter([]string{"cloudevents.type"}) stream, ok := view(metric.Instrument{Name: "http.server.request.duration"}) assert.True(t, ok, "view should match all instruments")