Implement OTEL secrets replacement in parsed policy

changelog/fragments/1771972977-Add-support-for-OTEL-secrets-handling.yaml

@@ -0,0 +1,33 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Add support for OTEL secrets handling
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Added functionality to replace secrets in OTEL sections (receivers, exporters, processors, extensions, connectors) of a policy.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: fleet-server
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/fleet-server/issues/6277

internal/pkg/policy/parsed_policy.go

@@ -115,6 +115,32 @@ func NewParsedPolicy(ctx context.Context, bulker bulk.Bulk, p model.Policy) (*Pa
 		secretKeys = append(secretKeys, "fleet."+key)
 	}
 
+	// Replace secrets in OTEL sections of policy
+	otelSections := []struct {
+		name string
+		data map[string]any
+	}{
+		{"receivers", p.Data.Receivers},
+		{"exporters", p.Data.Exporters},
+		{"processors", p.Data.Processors},
+		{"extensions", p.Data.Extensions},
+		{"connectors", p.Data.Connectors},
+	}
+	for _, section := range otelSections {
+		for componentName, component := range section.data {
+			if componentMap, ok := component.(map[string]any); ok {
+				ks, err := secret.ProcessMapSecrets(componentMap, secretValues)
+				if err != nil {
+					return nil, fmt.Errorf("failed to replace secrets in %s.%s section of policy: %w", section.name, componentName, err)
+				}
+				for _, key := range ks {
+					secretKeys = append(secretKeys, section.name+"."+componentName+"."+key)
+				}
+				section.data[componentName] = componentMap
+			}
+		}
+	}
+
 	// Done replacing secrets.
 	p.Data.SecretReferences = nil
 

internal/pkg/policy/parsed_policy_test.go

@@ -34,6 +34,9 @@ var testPolicyRemoteES string
 //go:embed testdata/policy_with_secrets_mixed.json
 var policyWithSecretsMixed string
 
+//go:embed testdata/policy_with_otel_secrets.json
+var policyWithOtelSecrets []byte
+
 func TestNewParsedPolicy(t *testing.T) {
 	// Run two formatting of the same payload to validate that the sha2 remains the same
 	testcases := []struct {
@@ -148,3 +151,45 @@ func TestParsedPolicyMixedSecretsReplacement(t *testing.T) {
 	require.Equal(t, "abcdef123_value", pp.Policy.Data.Fleet["hosts"].([]interface{})[0])
 	require.Equal(t, "w8yELZoBTAyw4gQK9KZ7_value", pp.Policy.Data.Fleet["ssl"].(map[string]interface{})["key"])
 }
+
+// TestParsedPolicyOTELSecretsReplacement tests that secrets in OTEL sections of a policy
+// (receivers, exporters, processors, extensions, connectors) are replaced correctly.
+func TestParsedPolicyOTELSecretsReplacement(t *testing.T) {
+	var m model.Policy
+	var d model.PolicyData
+	err := json.Unmarshal(policyWithOtelSecrets, &d)
+	require.NoError(t, err)
+
+	m.Data = &d
+
+	bulker := ftesting.NewMockBulk()
+	pp, err := NewParsedPolicy(t.Context(), bulker, m)
+	require.NoError(t, err)
+
+	// Validate that OTEL secret keys were identified
+	require.Contains(t, pp.SecretKeys, "receivers.otlp.auth")
+	require.Contains(t, pp.SecretKeys, "exporters.otlphttp/default.headers.authorization")
+	require.Contains(t, pp.SecretKeys, "processors.batch.api_key")
+	require.Contains(t, pp.SecretKeys, "extensions.basicauth.password")
+	require.Contains(t, pp.SecretKeys, "connectors.spanmetrics.token")
+
+	// Validate that inline secret references were replaced in receivers
+	otlpMap := pp.Policy.Data.Receivers["otlp"].(map[string]any)
+	require.Equal(t, "receiver-auth-id_value", otlpMap["auth"])
+
+	// Validate that path-based secret references were replaced in exporters
+	otlphttpMap := pp.Policy.Data.Exporters["otlphttp/default"].(map[string]any)
+	require.Equal(t, "exporter-auth-id_value", otlphttpMap["headers"].(map[string]any)["authorization"])
+
+	// Validate that inline secret references were replaced in processors
+	batchMap := pp.Policy.Data.Processors["batch"].(map[string]any)
+	require.Equal(t, "processor-key-id_value", batchMap["api_key"])
+
+	// Validate that path-based secret references were replaced in extensions
+	basicauthMap := pp.Policy.Data.Extensions["basicauth"].(map[string]any)
+	require.Equal(t, "extension-password-id_value", basicauthMap["password"])
+
+	// Validate that inline secret references were replaced in connectors
+	spanmetricsMap := pp.Policy.Data.Connectors["spanmetrics"].(map[string]any)
+	require.Equal(t, "connector-token-id_value", spanmetricsMap["token"])
+}

internal/pkg/policy/testdata/policy_with_otel_secrets.json

@@ -0,0 +1,72 @@
+{
+  "id": "otel-policy-id",
+  "revision": 1,
+  "outputs": {
+    "default": {
+      "type": "elasticsearch",
+      "hosts": [
+        "https://es.example.com:443"
+      ]
+    }
+  },
+  "output_permissions": {
+    "default": {
+      "_elastic_agent_monitoring": {
+        "indices": []
+      },
+      "_elastic_agent_checks": {
+        "cluster": [
+          "monitor"
+        ]
+      }
+    }
+  },
+  "receivers": {
+    "otlp": {
+      "auth": "$co.elastic.secret{receiver-auth-id}",
+      "protocols": {
+        "grpc": {
+          "endpoint": "0.0.0.0:4317"
+        }
+      }
+    }
+  },
+  "exporters": {
+    "otlphttp/default": {
+      "endpoint": "https://apm.example.com",
+      "secrets": {
+        "headers": {
+          "authorization": {
+            "id": "exporter-auth-id"
+          }
+        }
+      }
+    }
+  },
+  "processors": {
+    "batch": {
+      "api_key": "$co.elastic.secret{processor-key-id}"
+    }
+  },
+  "extensions": {
+    "basicauth": {
+      "secrets": {
+        "password": {
+          "id": "extension-password-id"
+        }
+      }
+    }
+  },
+  "connectors": {
+    "spanmetrics": {
+      "token": "$co.elastic.secret{connector-token-id}"
+    }
+  },
+  "secret_references": [
+    {"id": "receiver-auth-id"},
+    {"id": "exporter-auth-id"},
+    {"id": "processor-key-id"},
+    {"id": "extension-password-id"},
+    {"id": "connector-token-id"}
+  ]
+}