diff --git a/changelog/fragments/1771972977-Add-support-for-OTEL-secrets-handling.yaml b/changelog/fragments/1771972977-Add-support-for-OTEL-secrets-handling.yaml new file mode 100644 index 0000000000..c1c4ad83c6 --- /dev/null +++ b/changelog/fragments/1771972977-Add-support-for-OTEL-secrets-handling.yaml @@ -0,0 +1,33 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: bug-fix + +# Change summary; a 80ish characters long description of the change. +summary: Add support for OTEL secrets handling + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +description: | + Added functionality to replace secrets in OTEL sections (receivers, exporters, processors, extensions, connectors) of a policy. + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: fleet-server + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +issue: https://github.com/elastic/fleet-server/issues/6277 diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go index eb84c4f23c..8b817a2be1 100644 --- a/internal/pkg/policy/parsed_policy.go +++ b/internal/pkg/policy/parsed_policy.go @@ -115,6 +115,32 @@ func NewParsedPolicy(ctx context.Context, bulker bulk.Bulk, p model.Policy) (*Pa secretKeys = append(secretKeys, "fleet."+key) } + // Replace secrets in OTEL sections of policy + otelSections := []struct { + name string + data map[string]any + }{ + {"receivers", p.Data.Receivers}, + {"exporters", p.Data.Exporters}, + {"processors", p.Data.Processors}, + {"extensions", p.Data.Extensions}, + {"connectors", p.Data.Connectors}, + } + for _, section := range otelSections { + for componentName, component := range section.data { + if componentMap, ok := component.(map[string]any); ok { + ks, err := secret.ProcessMapSecrets(componentMap, secretValues) + if err != nil { + return nil, fmt.Errorf("failed to replace secrets in %s.%s section of policy: %w", section.name, componentName, err) + } + for _, key := range ks { + secretKeys = append(secretKeys, section.name+"."+componentName+"."+key) + } + section.data[componentName] = componentMap + } + } + } + // Done replacing secrets. p.Data.SecretReferences = nil diff --git a/internal/pkg/policy/parsed_policy_test.go b/internal/pkg/policy/parsed_policy_test.go index fbf15f5c0b..394f92b91a 100644 --- a/internal/pkg/policy/parsed_policy_test.go +++ b/internal/pkg/policy/parsed_policy_test.go @@ -34,6 +34,9 @@ var testPolicyRemoteES string //go:embed testdata/policy_with_secrets_mixed.json var policyWithSecretsMixed string +//go:embed testdata/policy_with_otel_secrets.json +var policyWithOtelSecrets []byte + func TestNewParsedPolicy(t *testing.T) { // Run two formatting of the same payload to validate that the sha2 remains the same testcases := []struct { @@ -148,3 +151,45 @@ func TestParsedPolicyMixedSecretsReplacement(t *testing.T) { require.Equal(t, "abcdef123_value", pp.Policy.Data.Fleet["hosts"].([]interface{})[0]) require.Equal(t, "w8yELZoBTAyw4gQK9KZ7_value", pp.Policy.Data.Fleet["ssl"].(map[string]interface{})["key"]) } + +// TestParsedPolicyOTELSecretsReplacement tests that secrets in OTEL sections of a policy +// (receivers, exporters, processors, extensions, connectors) are replaced correctly. +func TestParsedPolicyOTELSecretsReplacement(t *testing.T) { + var m model.Policy + var d model.PolicyData + err := json.Unmarshal(policyWithOtelSecrets, &d) + require.NoError(t, err) + + m.Data = &d + + bulker := ftesting.NewMockBulk() + pp, err := NewParsedPolicy(t.Context(), bulker, m) + require.NoError(t, err) + + // Validate that OTEL secret keys were identified + require.Contains(t, pp.SecretKeys, "receivers.otlp.auth") + require.Contains(t, pp.SecretKeys, "exporters.otlphttp/default.headers.authorization") + require.Contains(t, pp.SecretKeys, "processors.batch.api_key") + require.Contains(t, pp.SecretKeys, "extensions.basicauth.password") + require.Contains(t, pp.SecretKeys, "connectors.spanmetrics.token") + + // Validate that inline secret references were replaced in receivers + otlpMap := pp.Policy.Data.Receivers["otlp"].(map[string]any) + require.Equal(t, "receiver-auth-id_value", otlpMap["auth"]) + + // Validate that path-based secret references were replaced in exporters + otlphttpMap := pp.Policy.Data.Exporters["otlphttp/default"].(map[string]any) + require.Equal(t, "exporter-auth-id_value", otlphttpMap["headers"].(map[string]any)["authorization"]) + + // Validate that inline secret references were replaced in processors + batchMap := pp.Policy.Data.Processors["batch"].(map[string]any) + require.Equal(t, "processor-key-id_value", batchMap["api_key"]) + + // Validate that path-based secret references were replaced in extensions + basicauthMap := pp.Policy.Data.Extensions["basicauth"].(map[string]any) + require.Equal(t, "extension-password-id_value", basicauthMap["password"]) + + // Validate that inline secret references were replaced in connectors + spanmetricsMap := pp.Policy.Data.Connectors["spanmetrics"].(map[string]any) + require.Equal(t, "connector-token-id_value", spanmetricsMap["token"]) +} diff --git a/internal/pkg/policy/testdata/policy_with_otel_secrets.json b/internal/pkg/policy/testdata/policy_with_otel_secrets.json new file mode 100644 index 0000000000..a12bed0e9f --- /dev/null +++ b/internal/pkg/policy/testdata/policy_with_otel_secrets.json @@ -0,0 +1,72 @@ +{ + "id": "otel-policy-id", + "revision": 1, + "outputs": { + "default": { + "type": "elasticsearch", + "hosts": [ + "https://es.example.com:443" + ] + } + }, + "output_permissions": { + "default": { + "_elastic_agent_monitoring": { + "indices": [] + }, + "_elastic_agent_checks": { + "cluster": [ + "monitor" + ] + } + } + }, + "receivers": { + "otlp": { + "auth": "$co.elastic.secret{receiver-auth-id}", + "protocols": { + "grpc": { + "endpoint": "0.0.0.0:4317" + } + } + } + }, + "exporters": { + "otlphttp/default": { + "endpoint": "https://apm.example.com", + "secrets": { + "headers": { + "authorization": { + "id": "exporter-auth-id" + } + } + } + } + }, + "processors": { + "batch": { + "api_key": "$co.elastic.secret{processor-key-id}" + } + }, + "extensions": { + "basicauth": { + "secrets": { + "password": { + "id": "extension-password-id" + } + } + } + }, + "connectors": { + "spanmetrics": { + "token": "$co.elastic.secret{connector-token-id}" + } + }, + "secret_references": [ + {"id": "receiver-auth-id"}, + {"id": "exporter-auth-id"}, + {"id": "processor-key-id"}, + {"id": "extension-password-id"}, + {"id": "connector-token-id"} + ] +} diff --git a/internal/pkg/server/otel_policy_secrets_integration_test.go b/internal/pkg/server/otel_policy_secrets_integration_test.go new file mode 100644 index 0000000000..7dbee95df2 --- /dev/null +++ b/internal/pkg/server/otel_policy_secrets_integration_test.go @@ -0,0 +1,254 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration + +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/gofrs/uuid/v5" + "github.com/hashicorp/go-cleanhttp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/go-elasticsearch/v8" + + "github.com/elastic/fleet-server/v7/internal/pkg/api" + "github.com/elastic/fleet-server/v7/internal/pkg/apikey" + "github.com/elastic/fleet-server/v7/internal/pkg/bulk" + "github.com/elastic/fleet-server/v7/internal/pkg/dl" + "github.com/elastic/fleet-server/v7/internal/pkg/model" + testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log" +) + +// createAgentPolicyWithOtelSecrets creates two ES secrets, builds a policy that +// references them across all five OTEL sections, and returns the enrollment token +// for the new policy. All data-layer setup is done here so callers only need +// to perform the HTTP enroll/checkin flow. +func createAgentPolicyWithOtelSecrets(t *testing.T, ctx context.Context, bulker bulk.Bulk) string { + t.Helper() + + inlineSecretID := createSecret(t, ctx, bulker, "inline_secret_value") + inlineSecretRef := fmt.Sprintf("$co.elastic.secret{%s}", inlineSecretID) + pathSecretID := createSecret(t, ctx, bulker, "path_secret_value") + + policyID := uuid.Must(uuid.NewV4()).String() + var otelPolicyData = model.PolicyData{ + Outputs: map[string]map[string]any{ + "default": { + "type": "elasticsearch", + }, + }, + OutputPermissions: json.RawMessage(`{"default":{}}`), + Receivers: map[string]any{ + "otlp": map[string]any{ + "auth": inlineSecretRef, + }, + }, + // Exporter IDs must be "type/outputName"; only "elasticsearch" is supported. + Exporters: map[string]any{ + "elasticsearch/default": map[string]any{ + "secrets": map[string]any{ + "headers": map[string]any{ + "authorization": map[string]any{"id": pathSecretID}, + }, + }, + }, + }, + Processors: map[string]any{ + "batch": map[string]any{ + "api_key": inlineSecretRef, + }, + }, + Extensions: map[string]any{ + "basicauth": map[string]any{ + "secrets": map[string]any{ + "password": map[string]any{"id": pathSecretID}, + }, + }, + }, + Connectors: map[string]any{ + "spanmetrics": map[string]any{ + "token": inlineSecretRef, + }, + }, + SecretReferences: []model.SecretReferencesItems{ + {ID: inlineSecretID}, + {ID: pathSecretID}, + }, + } + + _, err := dl.CreatePolicy(ctx, bulker, model.Policy{ + PolicyID: policyID, + RevisionIdx: 1, + DefaultFleetServer: true, + Data: &otelPolicyData, + }) + if err != nil { + t.Fatal(err) + } + + esCfg := elasticsearch.Config{ + Username: "elastic", + Password: "changeme", + } + es, err := elasticsearch.NewClient(esCfg) + if err != nil { + t.Fatal(err) + } + key, err := apikey.Create(ctx, es, "default", "", "true", []byte(`{ + "fleet-apikey-enroll": { + "cluster": [], + "index": [], + "applications": [{ + "application": "fleet", + "privileges": ["no-privileges"], + "resources": ["*"] + }] + } + }`), map[string]any{ + "managed_by": "fleet", + "managed": true, + "type": "enroll", + "policy_id": policyID, + }) + if err != nil { + t.Fatal(err) + } + + _, err = dl.CreateEnrollmentAPIKey(ctx, bulker, model.EnrollmentAPIKey{ + Name: "Default", + APIKey: key.Key, + APIKeyID: key.ID, + PolicyID: policyID, + Active: true, + }) + if err != nil { + t.Fatal(err) + } + return key.Token() +} + +func Test_Agent_OtelPolicy_Secrets(t *testing.T) { + ctx := testlog.SetLogger(t).WithContext(t.Context()) + srv, err := startTestServer(t, ctx, policyData) + require.NoError(t, err) + + // Create secrets and policy with OTEL secret references before any agent interaction. + enrollKey := createAgentPolicyWithOtelSecrets(t, ctx, srv.bulker) + + cli := cleanhttp.DefaultClient() + + // enroll an agent + t.Log("Enroll an agent") + req, err := http.NewRequestWithContext(ctx, "POST", srv.baseURL()+"/api/fleet/agents/enroll", strings.NewReader(enrollBody)) + require.NoError(t, err) + req.Header.Set("Authorization", "ApiKey "+enrollKey) + req.Header.Set("User-Agent", "elastic agent "+serverVersion) + req.Header.Set("Content-Type", "application/json") + res, err := cli.Do(req) + require.NoError(t, err) + p, _ := io.ReadAll(res.Body) + res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode, "expected 200 OK return code, got %d: %s", res.StatusCode, string(p)) + t.Log("Agent enrollment successful") + + var obj map[string]any + err = json.Unmarshal(p, &obj) + require.NoError(t, err) + + item := obj["item"] + mm, ok := item.(map[string]any) + require.True(t, ok, "expected attribute item to be an object") + id := mm["id"] + agentID, ok := id.(string) + require.True(t, ok, "expected attribute id to be a string") + + apiKeyVal := mm["access_api_key"] + apiKey, ok := apiKeyVal.(string) + require.True(t, ok, "expected attribute access_api_key to be a string") + + // checkin + t.Logf("Fake a checkin for agent %s", agentID) + req, err = http.NewRequestWithContext(ctx, "POST", srv.baseURL()+"/api/fleet/agents/"+agentID+"/checkin", strings.NewReader(checkinBody)) + require.NoError(t, err) + req.Header.Set("Authorization", "ApiKey "+apiKey) + req.Header.Set("User-Agent", "elastic agent "+serverVersion) + req.Header.Set("Content-Type", "application/json") + res, err = cli.Do(req) + require.NoError(t, err) + body, _ := io.ReadAll(res.Body) + res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode, "expected checkin status to be 200 OK, got %d: %s", res.StatusCode, string(body)) + + t.Log("Checkin successful, verify body") + + var checkinResponse api.CheckinResponse + err = json.Unmarshal(body, &checkinResponse) + require.NoError(t, err) + + // expect 1 POLICY_CHANGE action + assert.Len(t, checkinResponse.Actions, 1) + assert.Equal(t, api.POLICYCHANGE, checkinResponse.Actions[0].Type) + actionData, err := checkinResponse.Actions[0].Data.AsActionPolicyChange() + require.NoError(t, err) + + // Assert receivers.otlp.auth was replaced with inline secret value + require.Contains(t, actionData.Policy.Receivers, "otlp") + otlpMap, ok := actionData.Policy.Receivers["otlp"].(map[string]any) + require.True(t, ok, "expected receivers.otlp to be a map") + assert.Equal(t, "inline_secret_value", otlpMap["auth"]) + + // Assert exporters.elasticsearch/default.headers.authorization was replaced with + // path secret value and the 'secrets' wrapper key removed. + // prepareOTelExporters also injects api_key from the prepared output. + require.Contains(t, actionData.Policy.Exporters, "elasticsearch/default") + esExporterMap, ok := actionData.Policy.Exporters["elasticsearch/default"].(map[string]any) + require.True(t, ok, "expected exporters.elasticsearch/default to be a map") + assert.NotContains(t, esExporterMap, "secrets", "expected 'secrets' key to be removed from exporters.elasticsearch/default") + require.Contains(t, esExporterMap, "headers") + headersMap, ok := esExporterMap["headers"].(map[string]any) + require.True(t, ok, "expected exporters.elasticsearch/default.headers to be a map") + assert.Equal(t, "path_secret_value", headersMap["authorization"]) + + // Assert processors.batch.api_key was replaced with inline secret value + require.Contains(t, actionData.Policy.Processors, "batch") + batchMap, ok := actionData.Policy.Processors["batch"].(map[string]any) + require.True(t, ok, "expected processors.batch to be a map") + assert.Equal(t, "inline_secret_value", batchMap["api_key"]) + + // Assert extensions.basicauth.password was replaced with path secret value + require.Contains(t, actionData.Policy.Extensions, "basicauth") + basicauthMap, ok := actionData.Policy.Extensions["basicauth"].(map[string]any) + require.True(t, ok, "expected extensions.basicauth to be a map") + assert.NotContains(t, basicauthMap, "secrets", "expected 'secrets' key to be removed from extensions.basicauth") + assert.Equal(t, "path_secret_value", basicauthMap["password"]) + + // Assert connectors.spanmetrics.token was replaced with inline secret value + require.Contains(t, actionData.Policy.Connectors, "spanmetrics") + spanmetricsMap, ok := actionData.Policy.Connectors["spanmetrics"].(map[string]any) + require.True(t, ok, "expected connectors.spanmetrics to be a map") + assert.Equal(t, "inline_secret_value", spanmetricsMap["token"]) + + // Assert secret_paths contains the expected OTEL keys + assert.ElementsMatch(t, + []string{ + "receivers.otlp.auth", + "exporters.elasticsearch/default.headers.authorization", + "processors.batch.api_key", + "extensions.basicauth.password", + "connectors.spanmetrics.token", + }, + actionData.Policy.SecretPaths, + ) + +}