Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions generator/resources/ec2_linux_test_matrix.json
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@
"instanceType": "t3a.medium",
"installAgentCommand": "go run ./install/install_agent.go rpm",
"ami": "cloudwatch-agent-integration-test-sles-15 20*",
"caCertPath": "/etc/ssl/certs/ca-bundle.crt",
"caCertPath": "/etc/ssl/ca-bundle.pem",
"arc": "amd64",
"binaryName": "amazon-cloudwatch-agent.rpm",
"family": "linux"
Expand All @@ -247,7 +247,7 @@
"instanceType": "m6g.medium",
"installAgentCommand": "go run ./install/install_agent.go rpm",
"ami": "cloudwatch-agent-integration-test-sles-15-arm64 20*",
"caCertPath": "/etc/ssl/certs/ca-bundle.crt",
"caCertPath": "/etc/ssl/ca-bundle.pem",
"arc": "arm64",
"binaryName": "amazon-cloudwatch-agent.rpm",
"family": "linux"
Expand Down
2 changes: 1 addition & 1 deletion terraform/eks/e2e/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ variable "instance_type" {

variable "helm_charts_branch" {
type = string
default = "main"
default = "f6a3940"
}

variable "cloudwatch_agent_repository" {
Expand Down
12 changes: 12 additions & 0 deletions test/ssm_document/ssm_document_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,13 @@ func Validate() error {
if err := awsservice.PutStringParameter(agentConfigFile1, agentConfig1); err != nil {
return err
}
// Parameter Store reads are eventually consistent: block until the
// just-written parameter is readable before issuing the configure command,
// otherwise the on-instance agent can fetch it before the write has
// propagated and fail with ParameterNotFound (a read-after-write race).
if err := awsservice.WaitForParameterAvailable(agentConfigFile1, 10*time.Second); err != nil {
return err
}

configureTest := testCase{
parameters: map[string][]string{
Expand All @@ -128,6 +135,11 @@ func Validate() error {
if err := awsservice.PutStringParameter(agentConfigFile2, agentConfig2); err != nil {
return err
}
// Same eventual-consistency race as agentConfigFile1 above: wait for the
// parameter to be readable before the configure-append command depends on it.
if err := awsservice.WaitForParameterAvailable(agentConfigFile2, 10*time.Second); err != nil {
return err
}

// Ensure SSM parameters are cleaned up
defer func() {
Expand Down
54 changes: 53 additions & 1 deletion util/awsservice/ssm.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,24 @@ func WaitForCommandCompletion(commandId, instanceId string) (*ssm.ListCommandInv

if len(result.CommandInvocations) > 0 {
invocation := result.CommandInvocations[0]
if invocation.Status == types.CommandInvocationStatusSuccess {
switch invocation.Status {
case types.CommandInvocationStatusSuccess:
return result, nil
case types.CommandInvocationStatusFailed,
types.CommandInvocationStatusCancelled,
types.CommandInvocationStatusTimedOut:
// Fail fast on a terminal non-success status instead of looping
// until the time budget expires and returning the generic
// "commands did not complete within 1 minute". A fast,
// deterministic command failure was previously masked as a 60s
// timeout; surfacing the real status and details here makes every
// future failure self-diagnosing.
details := ""
if invocation.StatusDetails != nil {
details = *invocation.StatusDetails
}
return nil, fmt.Errorf("command %s on instance %s reached terminal status %s: %s",
commandId, instanceId, invocation.Status, details)
}
}
}
Expand Down Expand Up @@ -129,6 +145,42 @@ func GetStringParameter(name string) string {
return *parameter.Parameter.Value
}

// GetParameter fetches a parameter value and returns the underlying error,
// unlike GetStringParameter which swallows the error and returns a sentinel
// string. This is needed for read-after-write checks where the caller must
// distinguish a genuine ParameterNotFound from a real value.
func GetParameter(name string) (string, error) {
parameter, err := SsmClient.GetParameter(ctx, &ssm.GetParameterInput{
Name: aws.String(name),
})
if err != nil {
return "", err
}

return *parameter.Parameter.Value, nil
}

// WaitForParameterAvailable polls Parameter Store until the named parameter is
// readable or the timeout elapses. SSM Parameter Store reads are eventually
// consistent, so a PutParameter is not guaranteed to be immediately visible to
// a subsequent read (e.g. an on-instance agent fetching the config). This
// bounded poll closes that read-after-write window before we issue a command
// that depends on the parameter, and fails closed (returns an error) if the
// parameter is still not readable when the budget is exhausted.
func WaitForParameterAvailable(name string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
var lastErr error
for time.Now().Before(deadline) {
if _, err := GetParameter(name); err == nil {
return nil
} else {
lastErr = err
}
time.Sleep(1 * time.Second)
}
return fmt.Errorf("parameter %q not readable within %v: %v", name, timeout, lastErr)
}

func putParameter(name, value string, paramType types.ParameterType) error {
isOverwriteAllowed := true

Expand Down
Loading