[test/otel/pernode] Add per-node + zero-step CRD E2E suite and EKS harness

terraform/eks/daemon/otel-pernode/main.tf

@@ -0,0 +1,253 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT
+#
+# E2E harness for the Target Allocator per-node allocation strategy plus zero-step
+# ServiceMonitor/PodMonitor CRD bundling (G1) and TA resilience to missing CRDs
+# (G2). Compared with ../otel (the standard suite) this harness:
+#   * installs a helm-charts checkout that BUNDLES the SM/PM CRDs (no separate
+#     prometheus-operator CRD install -- that is the whole point of G1),
+#   * deploys CUSTOM operator and Target Allocator images carrying the per-node +
+#     CRD-watch code (the public images do not have it),
+#   * forces the per-node allocation strategy on the AmazonCloudWatchAgent CR,
+#   * applies a ServiceMonitor/PodMonitor workload with a target_node relabel,
+#   * runs ./test/otel/pernode.
+
+module "common" {
+  source             = "../../../common"
+  cwagent_image_repo = var.cwagent_image_repo
+  cwagent_image_tag  = var.cwagent_image_tag
+}
+
+module "basic_components" {
+  source = "../../../basic_components"
+  region = var.region
+}
+
+locals {
+  aws_eks = "aws eks --region ${var.region}"
+}
+
+resource "aws_eks_cluster" "this" {
+  name     = "cwagent-eks-pernode-${module.common.testing_id}"
+  role_arn = module.basic_components.role_arn
+  version  = var.k8s_version
+  vpc_config {
+    subnet_ids         = module.basic_components.public_subnet_ids
+    security_group_ids = [module.basic_components.security_group]
+  }
+}
+
+# EKS Node Group -- >=2 nodes so per-node spread is meaningful.
+resource "aws_eks_node_group" "this" {
+  cluster_name    = aws_eks_cluster.this.name
+  node_group_name = "cwagent-pernode-node-${module.common.testing_id}"
+  node_role_arn   = aws_iam_role.node_role.arn
+  subnet_ids      = module.basic_components.public_subnet_ids
+
+  scaling_config {
+    desired_size = var.node_count
+    max_size     = var.node_count
+    min_size     = var.node_count
+  }
+
+  ami_type       = var.ami_type
+  capacity_type  = "ON_DEMAND"
+  disk_size      = 20
+  instance_types = [var.instance_type]
+
+  depends_on = [
+    aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly,
+    aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy,
+    aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy,
+  ]
+}
+
+# EKS Node IAM Role
+resource "aws_iam_role" "node_role" {
+  name = "cwagent-pernode-Worker-Role-${module.common.testing_id}"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Principal = { Service = "ec2.amazonaws.com" }
+      Action    = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "node_AmazonEKSWorkerNodePolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
+  role       = aws_iam_role.node_role.name
+}
+
+resource "aws_iam_role_policy_attachment" "node_AmazonEKS_CNI_Policy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
+  role       = aws_iam_role.node_role.name
+}
+
+resource "aws_iam_role_policy_attachment" "node_AmazonEC2ContainerRegistryReadOnly" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
+  role       = aws_iam_role.node_role.name
+}
+
+# Pod Identity IAM Role
+resource "aws_iam_role" "pod_identity_role" {
+  name = "cwagent-pernode-pod-identity-${module.common.testing_id}"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Principal = { Service = "pods.eks.amazonaws.com" }
+      Action    = ["sts:AssumeRole", "sts:TagSession"]
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "pod_identity_CloudWatchAgentServerPolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+  role       = aws_iam_role.pod_identity_role.name
+}
+
+# --- EKS Addon: Pod Identity agent ---
+
+resource "aws_eks_addon" "pod_identity_agent" {
+  depends_on   = [aws_eks_node_group.this]
+  cluster_name = aws_eks_cluster.this.name
+  addon_name   = "eks-pod-identity-agent"
+}
+
+# --- Update kubeconfig ---
+
+resource "null_resource" "kubectl" {
+  depends_on = [aws_eks_cluster.this, aws_eks_node_group.this]
+  provisioner "local-exec" {
+    command = "${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name}"
+  }
+}
+
+# --- Clone the helm-charts checkout that BUNDLES the SM/PM CRDs (G1). ---
+# NOTE: no prometheus-operator CRD install step exists in this harness on
+# purpose -- the SM/PM CRDs must come from the chart alone.
+
+data "external" "clone_helm_chart" {
+  count = var.local_chart_path == "" ? 1 : 0
+  program = ["bash", "-c", <<-EOT
+    rm -rf ./helm-charts
+    git clone -b ${var.helm_chart_branch} ${var.helm_chart_repo} ./helm-charts
+    echo '{"status":"ready"}'
+  EOT
+  ]
+}
+
+locals {
+  # Install from the local checkout when provided, else from the freshly cloned repo.
+  chart_path = var.local_chart_path != "" ? var.local_chart_path : "./helm-charts/charts/amazon-cloudwatch-observability"
+}
+
+resource "helm_release" "aws_observability" {
+  name             = "amazon-cloudwatch-observability"
+  chart            = local.chart_path
+  namespace        = "amazon-cloudwatch"
+  create_namespace = true
+  timeout          = 900
+
+  set = [
+    { name = "clusterName", value = aws_eks_cluster.this.name },
+    { name = "region", value = var.region },
+    { name = "otelContainerInsights.enabled", value = "true" },
+    # Custom operator image (carries the per-node strategy + CRD-watch resilience).
+    { name = "manager.image.repositoryDomainMap.public", value = var.operator_image_domain },
+    { name = "manager.image.repository", value = var.operator_image_repo },
+    { name = "manager.image.tag", value = var.operator_image_tag },
+  ]
+
+  depends_on = [
+    aws_eks_addon.pod_identity_agent,
+    null_resource.kubectl,
+    data.external.clone_helm_chart,
+  ]
+}
+
+# --- Pod Identity association (after Helm creates the service account) ---
+
+resource "aws_eks_pod_identity_association" "cloudwatch_agent" {
+  depends_on      = [helm_release.aws_observability]
+  cluster_name    = aws_eks_cluster.this.name
+  namespace       = "amazon-cloudwatch"
+  service_account = "cloudwatch-agent"
+  role_arn        = aws_iam_role.pod_identity_role.arn
+}
+
+# --- Patch the CR: custom agent image + custom Target Allocator image + force
+#     the per-node allocation strategy. The TA image is not a first-class chart
+#     value, so it is patched onto the AmazonCloudWatchAgent CR directly
+#     (mirrors scripts/deploy-all.sh CUSTOM_TA). ---
+
+resource "null_resource" "patch_cr" {
+  depends_on = [helm_release.aws_observability, null_resource.kubectl]
+  triggers   = { timestamp = timestamp() }
+  provisioner "local-exec" {
+    command = <<-EOT
+      set -e
+      sleep 30
+      kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type='json' \
+        -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]'
+      kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type=merge \
+        -p='{"spec":{"targetAllocator":{"allocationStrategy":"${var.allocation_strategy}","image":"${var.ta_image}"}}}'
+      sleep 10
+    EOT
+  }
+}
+
+# --- Restart pods to pick up Pod Identity + new images ---
+
+resource "null_resource" "restart_pods" {
+  depends_on = [aws_eks_pod_identity_association.cloudwatch_agent, null_resource.patch_cr]
+  triggers   = { timestamp = timestamp() }
+  provisioner "local-exec" {
+    command = <<-EOT
+      kubectl -n amazon-cloudwatch rollout restart daemonset/cloudwatch-agent
+      kubectl -n amazon-cloudwatch rollout restart deployment/cloudwatch-agent-target-allocator 2>/dev/null || true
+      kubectl -n amazon-cloudwatch rollout status daemonset/cloudwatch-agent --timeout=180s
+      kubectl -n amazon-cloudwatch rollout status deployment/cloudwatch-agent-target-allocator --timeout=180s 2>/dev/null || true
+    EOT
+  }
+}
+
+# --- Apply the per-node workload (ServiceMonitor + PodMonitor + load gen). The
+#     SM/PM CRs depend on the CRDs the chart bundled. ---
+
+resource "null_resource" "workload" {
+  depends_on = [helm_release.aws_observability, null_resource.restart_pods]
+  triggers   = { timestamp = timestamp() }
+  provisioner "local-exec" {
+    command = "kubectl apply -f ${path.module}/../../../../test/otel/pernode/resources/workload.yaml"
+  }
+}
+
+# --- Test runner ---
+
+resource "null_resource" "validator" {
+  depends_on = [
+    null_resource.restart_pods,
+    null_resource.workload,
+  ]
+
+  triggers = { always_run = timestamp() }
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      echo "Running OTEL per-node integration tests"
+      cd ../../../..
+
+      echo "Waiting 3 minutes for metrics to propagate..."
+      sleep 180
+
+      go test -tags integration -timeout 1h -v ${var.test_dir} \
+        -eksClusterName=${aws_eks_cluster.this.name} \
+        -computeType=EKS \
+        -eksDeploymentStrategy=DAEMON \
+        -region=${var.region}
+    EOT
+  }
+}

terraform/eks/daemon/otel-pernode/providers.tf

@@ -0,0 +1,28 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+provider "aws" {
+  region = var.region
+}
+
+provider "kubernetes" {
+  host                   = aws_eks_cluster.this.endpoint
+  cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data)
+  exec {
+    api_version = "client.authentication.k8s.io/v1beta1"
+    command     = "aws"
+    args        = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name]
+  }
+}
+
+provider "helm" {
+  kubernetes = {
+    host                   = aws_eks_cluster.this.endpoint
+    cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data)
+    exec = {
+      api_version = "client.authentication.k8s.io/v1beta1"
+      command     = "aws"
+      args        = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name]
+    }
+  }
+}

terraform/eks/daemon/otel-pernode/variables.tf

@@ -0,0 +1,97 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+variable "region" {
+  type    = string
+  default = "us-west-2"
+}
+
+variable "test_dir" {
+  type    = string
+  default = "./test/otel/pernode"
+}
+
+# --- Agent image (DaemonSet). Public by default; per-node needs no agent change. ---
+variable "cwagent_image_repo" {
+  type    = string
+  default = "public.ecr.aws/cloudwatch-agent/cloudwatch-agent"
+}
+
+variable "cwagent_image_tag" {
+  type    = string
+  default = "latest"
+}
+
+# --- Helm chart source. MUST point at a checkout that contains the zero-step CRD
+#     bundling (G1) changes. Defaults to this repo's fork; override for a PR branch. ---
+variable "helm_chart_repo" {
+  type    = string
+  default = "https://github.com/aws-observability/helm-charts.git"
+}
+
+variable "helm_chart_branch" {
+  type    = string
+  default = "main"
+}
+
+# Install from a LOCAL chart checkout instead of git-cloning helm_chart_repo.
+# Useful to test working-tree changes that are not yet committed/pushed. When
+# set (absolute path to .../charts/amazon-cloudwatch-observability), the clone is
+# skipped. Leave empty for CI (clone).
+variable "local_chart_path" {
+  type    = string
+  default = ""
+}
+
+# --- Custom operator image: REQUIRED. The public operator hardcodes
+#     consistent-hashing and ignores the per-node CR field, so the per-node +
+#     CRD-watch (G2) code only runs from a custom build. ---
+variable "operator_image_domain" {
+  type        = string
+  description = "Registry domain for the custom operator image (maps manager.image.repositoryDomainMap.public)."
+  # e.g. <account>.dkr.ecr.us-west-2.amazonaws.com
+}
+
+variable "operator_image_repo" {
+  type        = string
+  description = "Repository (path) for the custom operator image, e.g. wenepra/cwagent-test/cloudwatch-agent-operator."
+}
+
+variable "operator_image_tag" {
+  type = string
+}
+
+# --- Custom Target Allocator image: REQUIRED. Patched onto the
+#     AmazonCloudWatchAgent CR after install (the chart does not expose it as a
+#     first-class value), mirroring scripts/deploy-all.sh CUSTOM_TA. ---
+variable "ta_image" {
+  type        = string
+  description = "Full Target Allocator image ref, e.g. <ecr>/cloudwatch-agent-target-allocator:pernodeN."
+}
+
+# --- Allocation strategy under test. ---
+variable "allocation_strategy" {
+  type    = string
+  default = "per-node"
+}
+
+variable "k8s_version" {
+  type    = string
+  default = "1.35"
+}
+
+variable "ami_type" {
+  type    = string
+  default = "AL2023_x86_64_STANDARD"
+}
+
+variable "instance_type" {
+  type    = string
+  default = "t3.medium"
+}
+
+# Number of worker nodes. >=2 so per-node spread is meaningful.
+variable "node_count" {
+  type    = number
+  default = 2
+}