From 3a29e8e21ca54f7e385ed70b53ae7bba16012397 Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Wed, 8 Jul 2026 16:03:15 +0000 Subject: [PATCH 1/2] Fix Prometheus escape for OTel expandconverter --- .../prometheus/dollar_escape_test.go | 36 +++++++++++++++ .../opentelemetry/prometheus/translator.go | 24 +++++++--- .../prometheus/translator_test.go | 45 +++++++++++++++++++ 3 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 translator/translate/otel/pipeline/opentelemetry/prometheus/dollar_escape_test.go diff --git a/translator/translate/otel/pipeline/opentelemetry/prometheus/dollar_escape_test.go b/translator/translate/otel/pipeline/opentelemetry/prometheus/dollar_escape_test.go new file mode 100644 index 0000000000..3aebdf851a --- /dev/null +++ b/translator/translate/otel/pipeline/opentelemetry/prometheus/dollar_escape_test.go @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package prometheus + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEscapeDollarDigit(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {name: "bare $1", in: "replacement: $1", want: "replacement: $$1"}, + {name: "bare $0", in: "$0", want: "$$0"}, + {name: "bare $9", in: "value=$9", want: "value=$$9"}, + {name: "multiple refs", in: "$1 and $2 and $3", want: "$$1 and $$2 and $$3"}, + {name: "already escaped $$1", in: "$$1", want: "$$$1"}, + {name: "dollar non-digit", in: "$HOME and $PATH", want: "$HOME and $PATH"}, + {name: "no dollar", in: "no dollars here", want: "no dollars here"}, + {name: "empty string", in: "", want: ""}, + {name: "dollar at end", in: "trailing$", want: "trailing$"}, + {name: "multi-digit $10", in: "$10", want: "$$10"}, + {name: "mixed text", in: "tag: k8s.label.$1 and $FOO", want: "tag: k8s.label.$$1 and $FOO"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := escapeDollarDigit(tt.in) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/translator/translate/otel/pipeline/opentelemetry/prometheus/translator.go b/translator/translate/otel/pipeline/opentelemetry/prometheus/translator.go index 920e17d421..1abf73e513 100644 --- a/translator/translate/otel/pipeline/opentelemetry/prometheus/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/prometheus/translator.go @@ -103,18 +103,14 @@ func (t *prometheusReceiverTranslator) Translate(conf *confmap.Conf) (component. return nil, fmt.Errorf("unable to read prometheus config from path %s: %w", configPath, err) } + // Prevent OTel expandconverter from misinterpreting Prometheus regex backreferences. + escaped := escapeDollarDigit(string(content)) var stringMap map[string]interface{} - if err := yaml.Unmarshal(content, &stringMap); err != nil { + if err := yaml.Unmarshal([]byte(escaped), &stringMap); err != nil { return nil, fmt.Errorf("unable to parse prometheus config from %s: %w", configPath, err) } componentParser := confmap.NewFromStringMap(stringMap) - // NOTE: Prometheus relabel configs use $1, $2 for capture group references. - // The OTel confmap expandconverter interprets these as environment variable - // references (os.Expand), which can cause failures or empty replacements. - // This is a known limitation when prometheus configs with relabel_configs - // are loaded through the OTel config resolver pipeline. - var promCfg prometheusreceiver.PromConfig if err := componentParser.Unmarshal(&promCfg); err != nil { return nil, fmt.Errorf("unable to unmarshal prometheus config from %s: %w", configPath, err) @@ -125,3 +121,17 @@ func (t *prometheusReceiverTranslator) Translate(conf *confmap.Conf) (component. return cfg, nil } + +// escapeDollarDigit escapes bare $ followed by a digit so the expandconverter +// does not interpret regex backreferences (e.g., $1 in relabel_configs) as env vars. +func escapeDollarDigit(s string) string { + var out []byte + for i := 0; i < len(s); i++ { + if s[i] == '$' && i+1 < len(s) && s[i+1] >= '0' && s[i+1] <= '9' { + out = append(out, '$', '$') + } else { + out = append(out, s[i]) + } + } + return string(out) +} diff --git a/translator/translate/otel/pipeline/opentelemetry/prometheus/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/prometheus/translator_test.go index 0fe6c31a75..78e50a3559 100644 --- a/translator/translate/otel/pipeline/opentelemetry/prometheus/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/prometheus/translator_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/confmap" @@ -162,6 +163,50 @@ func TestPrometheusReceiverTranslator(t *testing.T) { assert.NotNil(t, cfg) } +func TestPrometheusReceiverTranslatorDollarEscape(t *testing.T) { + // Create a prometheus config with relabel_configs that use $1 capture group references. + // Without escaping, the OTel expandconverter would treat $1 as an env var and blank it out. + content := []byte(`scrape_configs: + - job_name: k8s_pods + relabel_configs: + - source_labels: [__meta_kubernetes_pod_name] + regex: (.*) + target_label: pod + replacement: $1 + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + regex: ([^:]+)(?::\d+)?;(\d+) + target_label: __address__ + replacement: $1:$2 +`) + dir := t.TempDir() + path := filepath.Join(dir, "prometheus_relabel.yml") + require.NoError(t, os.WriteFile(path, content, 0600)) + + conf := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "prometheus": map[string]interface{}{ + "config_path": path, + }, + }, + }, + }) + + receiver := &prometheusReceiverTranslator{} + cfg, err := receiver.Translate(conf) + require.NoError(t, err) + + promCfg := cfg.(*prometheusreceiver.Config) + require.Len(t, promCfg.PrometheusConfig.ScrapeConfigs, 1) + + relabelCfgs := promCfg.PrometheusConfig.ScrapeConfigs[0].RelabelConfigs + require.Len(t, relabelCfgs, 2) + // $$1 is the correct value at this stage — the expandconverter resolves $$1 → $1 at + // collector startup. Without the escape, $1 would be treated as an env var and blanked out. + assert.Equal(t, "$$1", relabelCfgs[0].Replacement, "first relabel $1 should be escaped to $$1 for expandconverter") + assert.Equal(t, "$$1:$$2", relabelCfgs[1].Replacement, "second relabel $1:$2 should be escaped to $$1:$$2 for expandconverter") +} + func TestPrometheusReceiverTranslatorMissingFile(t *testing.T) { conf := confmap.NewFromStringMap(map[string]interface{}{ "opentelemetry": map[string]interface{}{ From 8a86699e6827b1fe8db66c7638edd706caebec7d Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Wed, 8 Jul 2026 18:34:28 +0000 Subject: [PATCH 2/2] Add escape dollar feature to prometheus golden yaml sample --- .../combined_v1_v2_ec2_config.yaml | 16 +++++++++++---- .../combined_v1_v2_eks_config.yaml | 20 +++++++++++++------ .../prometheus_otel_pipeline_config.yaml | 8 ++++++++ .../prometheus_otel_pipeline_eks_config.yaml | 8 ++++++++ .../prometheus_otel_scrape_config.yaml | 5 +++++ translator/tocwconfig/tocwconfig_unix_test.go | 1 + 6 files changed, 48 insertions(+), 10 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml index ba522dcec8..2cab97f753 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml @@ -1386,6 +1386,14 @@ receivers: honor_timestamps: true job_name: test_app metrics_path: /metrics + relabel_configs: + - action: replace + regex: (.*) + replacement: $$1 + separator: ; + source_labels: + - __meta_kubernetes_pod_name + target_label: pod scheme: http scrape_interval: 30s scrape_protocols: @@ -1553,12 +1561,12 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_cpu - - telegraf_processes - telegraf_mem - telegraf_netstat - telegraf_disk - telegraf_procstat/1917393364 + - telegraf_cpu + - telegraf_processes - telegraf_swap metrics/host_metrics: exporters: @@ -1574,8 +1582,8 @@ service: - ec2tagger - awsentity/service/telegraf receivers: - - telegraf_statsd - telegraf_socket_listener + - telegraf_statsd metrics/hostDeltaMetrics: exporters: - awscloudwatch @@ -1584,8 +1592,8 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_net - telegraf_diskio + - telegraf_net metrics/opentelemetry: exporters: - otlphttp/metrics diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index 84971c6654..a4f774caf1 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -587,9 +587,9 @@ processors: ec2_instance_tag_keys: - AutoScalingGroupName ec2_metadata_tags: + - InstanceType - ImageId - InstanceId - - InstanceType imds_retries: 1 middleware: agenthealth/statuscode refresh_tags_interval: 0s @@ -2081,6 +2081,14 @@ receivers: honor_timestamps: true job_name: test_app metrics_path: /metrics + relabel_configs: + - action: replace + regex: (.*) + replacement: $$1 + separator: ; + source_labels: + - __meta_kubernetes_pod_name + target_label: pod scheme: http scrape_interval: 30s scrape_protocols: @@ -2418,13 +2426,13 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_swap - - telegraf_mem + - telegraf_procstat/1917393364 - telegraf_processes - - telegraf_netstat - telegraf_disk - - telegraf_procstat/1917393364 + - telegraf_mem - telegraf_cpu + - telegraf_swap + - telegraf_netstat metrics/host_metrics: exporters: - forward/opentelemetry @@ -2449,8 +2457,8 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_net - telegraf_diskio + - telegraf_net metrics/opentelemetry: exporters: - otlphttp/metrics diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_pipeline_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_pipeline_config.yaml index c2c536aa60..f4be5ec0c0 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_pipeline_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_pipeline_config.yaml @@ -390,6 +390,14 @@ receivers: honor_timestamps: true job_name: test_app metrics_path: /metrics + relabel_configs: + - action: replace + regex: (.*) + replacement: $$1 + separator: ; + source_labels: + - __meta_kubernetes_pod_name + target_label: pod scheme: http scrape_interval: 30s scrape_protocols: diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_pipeline_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_pipeline_eks_config.yaml index dd01e4d13d..d336f409e2 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_pipeline_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_pipeline_eks_config.yaml @@ -552,6 +552,14 @@ receivers: honor_timestamps: true job_name: test_app metrics_path: /metrics + relabel_configs: + - action: replace + regex: (.*) + replacement: $$1 + separator: ; + source_labels: + - __meta_kubernetes_pod_name + target_label: pod scheme: http scrape_interval: 30s scrape_protocols: diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_scrape_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_scrape_config.yaml index a78addd379..54e56c8b8e 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_scrape_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/prometheus_otel_scrape_config.yaml @@ -5,3 +5,8 @@ scrape_configs: - job_name: 'test_app' static_configs: - targets: ['localhost:9090'] + relabel_configs: + - source_labels: [__meta_kubernetes_pod_name] + regex: (.*) + target_label: pod + replacement: $1 diff --git a/translator/tocwconfig/tocwconfig_unix_test.go b/translator/tocwconfig/tocwconfig_unix_test.go index ee5d7022c3..4895852ec5 100644 --- a/translator/tocwconfig/tocwconfig_unix_test.go +++ b/translator/tocwconfig/tocwconfig_unix_test.go @@ -176,6 +176,7 @@ func TestCombinedV1V2EKSConfig(t *testing.T) { yamlConfig, err := mapstructure.Marshal(cfg) require.NoError(t, err) yamlStr := toyamlconfig.ToYamlConfig(yamlConfig) + // assert.NoError(t, os.WriteFile("./sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml", []byte(yamlStr), 0644)) // useful for regenerating YAML require.NoError(t, yaml.Unmarshal([]byte(yamlStr), &actual)) opt := cmpopts.SortSlices(func(x, y interface{}) bool {