diff --git a/generator/resources/ec2_linux_test_matrix.json b/generator/resources/ec2_linux_test_matrix.json index 832ec5afe..941ecff62 100644 --- a/generator/resources/ec2_linux_test_matrix.json +++ b/generator/resources/ec2_linux_test_matrix.json @@ -236,7 +236,7 @@ "instanceType": "t3a.medium", "installAgentCommand": "go run ./install/install_agent.go rpm", "ami": "cloudwatch-agent-integration-test-sles-15 20*", - "caCertPath": "/etc/ssl/certs/ca-bundle.crt", + "caCertPath": "/etc/ssl/ca-bundle.pem", "arc": "amd64", "binaryName": "amazon-cloudwatch-agent.rpm", "family": "linux" @@ -247,7 +247,7 @@ "instanceType": "m6g.medium", "installAgentCommand": "go run ./install/install_agent.go rpm", "ami": "cloudwatch-agent-integration-test-sles-15-arm64 20*", - "caCertPath": "/etc/ssl/certs/ca-bundle.crt", + "caCertPath": "/etc/ssl/ca-bundle.pem", "arc": "arm64", "binaryName": "amazon-cloudwatch-agent.rpm", "family": "linux" diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 96f96b887..30b29e5dc 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -33,7 +33,7 @@ variable "instance_type" { variable "helm_charts_branch" { type = string - default = "main" + default = "f6a3940" } variable "cloudwatch_agent_repository" { diff --git a/test/ssm_document/ssm_document_unix.go b/test/ssm_document/ssm_document_unix.go index 443f4ad2e..0b1512d57 100644 --- a/test/ssm_document/ssm_document_unix.go +++ b/test/ssm_document/ssm_document_unix.go @@ -108,6 +108,13 @@ func Validate() error { if err := awsservice.PutStringParameter(agentConfigFile1, agentConfig1); err != nil { return err } + // Parameter Store reads are eventually consistent: block until the + // just-written parameter is readable before issuing the configure command, + // otherwise the on-instance agent can fetch it before the write has + // propagated and fail with ParameterNotFound (a read-after-write race). + if err := awsservice.WaitForParameterAvailable(agentConfigFile1, 10*time.Second); err != nil { + return err + } configureTest := testCase{ parameters: map[string][]string{ @@ -128,6 +135,11 @@ func Validate() error { if err := awsservice.PutStringParameter(agentConfigFile2, agentConfig2); err != nil { return err } + // Same eventual-consistency race as agentConfigFile1 above: wait for the + // parameter to be readable before the configure-append command depends on it. + if err := awsservice.WaitForParameterAvailable(agentConfigFile2, 10*time.Second); err != nil { + return err + } // Ensure SSM parameters are cleaned up defer func() { diff --git a/util/awsservice/ssm.go b/util/awsservice/ssm.go index d13b11505..e36f52449 100644 --- a/util/awsservice/ssm.go +++ b/util/awsservice/ssm.go @@ -99,8 +99,24 @@ func WaitForCommandCompletion(commandId, instanceId string) (*ssm.ListCommandInv if len(result.CommandInvocations) > 0 { invocation := result.CommandInvocations[0] - if invocation.Status == types.CommandInvocationStatusSuccess { + switch invocation.Status { + case types.CommandInvocationStatusSuccess: return result, nil + case types.CommandInvocationStatusFailed, + types.CommandInvocationStatusCancelled, + types.CommandInvocationStatusTimedOut: + // Fail fast on a terminal non-success status instead of looping + // until the time budget expires and returning the generic + // "commands did not complete within 1 minute". A fast, + // deterministic command failure was previously masked as a 60s + // timeout; surfacing the real status and details here makes every + // future failure self-diagnosing. + details := "" + if invocation.StatusDetails != nil { + details = *invocation.StatusDetails + } + return nil, fmt.Errorf("command %s on instance %s reached terminal status %s: %s", + commandId, instanceId, invocation.Status, details) } } } @@ -129,6 +145,42 @@ func GetStringParameter(name string) string { return *parameter.Parameter.Value } +// GetParameter fetches a parameter value and returns the underlying error, +// unlike GetStringParameter which swallows the error and returns a sentinel +// string. This is needed for read-after-write checks where the caller must +// distinguish a genuine ParameterNotFound from a real value. +func GetParameter(name string) (string, error) { + parameter, err := SsmClient.GetParameter(ctx, &ssm.GetParameterInput{ + Name: aws.String(name), + }) + if err != nil { + return "", err + } + + return *parameter.Parameter.Value, nil +} + +// WaitForParameterAvailable polls Parameter Store until the named parameter is +// readable or the timeout elapses. SSM Parameter Store reads are eventually +// consistent, so a PutParameter is not guaranteed to be immediately visible to +// a subsequent read (e.g. an on-instance agent fetching the config). This +// bounded poll closes that read-after-write window before we issue a command +// that depends on the parameter, and fails closed (returns an error) if the +// parameter is still not readable when the budget is exhausted. +func WaitForParameterAvailable(name string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + if _, err := GetParameter(name); err == nil { + return nil + } else { + lastErr = err + } + time.Sleep(1 * time.Second) + } + return fmt.Errorf("parameter %q not readable within %v: %v", name, timeout, lastErr) +} + func putParameter(name, value string, paramType types.ParameterType) error { isOverwriteAllowed := true