From af8785f1636fa79063f355e60f2e55447022fed1 Mon Sep 17 00:00:00 2001 From: Musa Asad Date: Wed, 17 Jun 2026 16:27:50 +0000 Subject: [PATCH] test: pin EKS E2E helm-charts ref and fix EC2 Linux integration tests EKS E2E helm-charts pin ----------------------- The EKS E2E module cloned aws-observability/helm-charts at a floating "main" ref. A dependency bump renamed the operator's registered feature-gate IDs from hyphenated to non-hyphenated forms; the chart kept passing the old hyphenated IDs on the operator's --feature-gates argument until it was re-aligned in chart PR #318 (commit f6a3940). During that window the E2E Helm path launched the post-bump operator with two unregistered gate IDs, which made the controller-manager exit at startup and CrashLoop, timing out the readiness wait and failing terraform apply. Pinning helm_charts_branch to f6a3940 removes the cross-repo desync window so the cloned chart is always a fixed, reviewed version aligned with the operator's gate IDs. EC2 Linux integration test fixes -------------------------------- Three EC2 Linux integration tests were failing on recent CI runs. Two are fixed here; the third is documented as an infra/AMI issue. 1. sles-15 ca_bundle_test: the two sles-15 rows in generator/resources/ec2_linux_test_matrix.json hard-coded the RHEL CA bundle path /etc/ssl/certs/ca-bundle.crt, which does not exist on SLES. Set both sles-15 rows to the SUSE path /etc/ssl/ca-bundle.pem. RHEL / Alma / Rocky rows are unchanged. 2. alma-linux-10 ssm_document_test: two harness hardening edits. - util/awsservice/ssm.go WaitForCommandCompletion now returns immediately on a terminal non-success status (Failed / Cancelled / TimedOut) with the real status and status details, instead of looping until the budget expires and returning the generic "commands did not complete within 1 minute". This stops a fast, deterministic command failure from being misreported as a 60s timeout. - test/ssm_document/ssm_document_unix.go adds a bounded read-after-write poll (new WaitForParameterAvailable helper) after each PutStringParameter and before the dependent configure SendCommand, so the on-instance agent cannot fetch the config before the write has propagated (Parameter Store reads are eventually consistent). 3. rocky-linux-9 ssm_document_test: this is an infra/AMI issue, not a test bug. The instance never registers as an Online SSM managed instance within the readiness gate. The gate timeout was intentionally NOT raised (it would not help a never-registering instance). No code fix is forced; see the PR description for details and the recommendation to keep rocky-9 non-blocking. --- .../resources/ec2_linux_test_matrix.json | 4 +- terraform/eks/e2e/variables.tf | 2 +- test/ssm_document/ssm_document_unix.go | 12 +++++ util/awsservice/ssm.go | 54 ++++++++++++++++++- 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/generator/resources/ec2_linux_test_matrix.json b/generator/resources/ec2_linux_test_matrix.json index 832ec5afe..941ecff62 100644 --- a/generator/resources/ec2_linux_test_matrix.json +++ b/generator/resources/ec2_linux_test_matrix.json @@ -236,7 +236,7 @@ "instanceType": "t3a.medium", "installAgentCommand": "go run ./install/install_agent.go rpm", "ami": "cloudwatch-agent-integration-test-sles-15 20*", - "caCertPath": "/etc/ssl/certs/ca-bundle.crt", + "caCertPath": "/etc/ssl/ca-bundle.pem", "arc": "amd64", "binaryName": "amazon-cloudwatch-agent.rpm", "family": "linux" @@ -247,7 +247,7 @@ "instanceType": "m6g.medium", "installAgentCommand": "go run ./install/install_agent.go rpm", "ami": "cloudwatch-agent-integration-test-sles-15-arm64 20*", - "caCertPath": "/etc/ssl/certs/ca-bundle.crt", + "caCertPath": "/etc/ssl/ca-bundle.pem", "arc": "arm64", "binaryName": "amazon-cloudwatch-agent.rpm", "family": "linux" diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 96f96b887..30b29e5dc 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -33,7 +33,7 @@ variable "instance_type" { variable "helm_charts_branch" { type = string - default = "main" + default = "f6a3940" } variable "cloudwatch_agent_repository" { diff --git a/test/ssm_document/ssm_document_unix.go b/test/ssm_document/ssm_document_unix.go index 443f4ad2e..0b1512d57 100644 --- a/test/ssm_document/ssm_document_unix.go +++ b/test/ssm_document/ssm_document_unix.go @@ -108,6 +108,13 @@ func Validate() error { if err := awsservice.PutStringParameter(agentConfigFile1, agentConfig1); err != nil { return err } + // Parameter Store reads are eventually consistent: block until the + // just-written parameter is readable before issuing the configure command, + // otherwise the on-instance agent can fetch it before the write has + // propagated and fail with ParameterNotFound (a read-after-write race). + if err := awsservice.WaitForParameterAvailable(agentConfigFile1, 10*time.Second); err != nil { + return err + } configureTest := testCase{ parameters: map[string][]string{ @@ -128,6 +135,11 @@ func Validate() error { if err := awsservice.PutStringParameter(agentConfigFile2, agentConfig2); err != nil { return err } + // Same eventual-consistency race as agentConfigFile1 above: wait for the + // parameter to be readable before the configure-append command depends on it. + if err := awsservice.WaitForParameterAvailable(agentConfigFile2, 10*time.Second); err != nil { + return err + } // Ensure SSM parameters are cleaned up defer func() { diff --git a/util/awsservice/ssm.go b/util/awsservice/ssm.go index d13b11505..e36f52449 100644 --- a/util/awsservice/ssm.go +++ b/util/awsservice/ssm.go @@ -99,8 +99,24 @@ func WaitForCommandCompletion(commandId, instanceId string) (*ssm.ListCommandInv if len(result.CommandInvocations) > 0 { invocation := result.CommandInvocations[0] - if invocation.Status == types.CommandInvocationStatusSuccess { + switch invocation.Status { + case types.CommandInvocationStatusSuccess: return result, nil + case types.CommandInvocationStatusFailed, + types.CommandInvocationStatusCancelled, + types.CommandInvocationStatusTimedOut: + // Fail fast on a terminal non-success status instead of looping + // until the time budget expires and returning the generic + // "commands did not complete within 1 minute". A fast, + // deterministic command failure was previously masked as a 60s + // timeout; surfacing the real status and details here makes every + // future failure self-diagnosing. + details := "" + if invocation.StatusDetails != nil { + details = *invocation.StatusDetails + } + return nil, fmt.Errorf("command %s on instance %s reached terminal status %s: %s", + commandId, instanceId, invocation.Status, details) } } } @@ -129,6 +145,42 @@ func GetStringParameter(name string) string { return *parameter.Parameter.Value } +// GetParameter fetches a parameter value and returns the underlying error, +// unlike GetStringParameter which swallows the error and returns a sentinel +// string. This is needed for read-after-write checks where the caller must +// distinguish a genuine ParameterNotFound from a real value. +func GetParameter(name string) (string, error) { + parameter, err := SsmClient.GetParameter(ctx, &ssm.GetParameterInput{ + Name: aws.String(name), + }) + if err != nil { + return "", err + } + + return *parameter.Parameter.Value, nil +} + +// WaitForParameterAvailable polls Parameter Store until the named parameter is +// readable or the timeout elapses. SSM Parameter Store reads are eventually +// consistent, so a PutParameter is not guaranteed to be immediately visible to +// a subsequent read (e.g. an on-instance agent fetching the config). This +// bounded poll closes that read-after-write window before we issue a command +// that depends on the parameter, and fails closed (returns an error) if the +// parameter is still not readable when the budget is exhausted. +func WaitForParameterAvailable(name string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + if _, err := GetParameter(name); err == nil { + return nil + } else { + lastErr = err + } + time.Sleep(1 * time.Second) + } + return fmt.Errorf("parameter %q not readable within %v: %v", name, timeout, lastErr) +} + func putParameter(name, value string, paramType types.ParameterType) error { isOverwriteAllowed := true