diff --git a/generator/resources/eks_daemon_test_matrix.json b/generator/resources/eks_daemon_test_matrix.json index 245c75b16..e11e85915 100644 --- a/generator/resources/eks_daemon_test_matrix.json +++ b/generator/resources/eks_daemon_test_matrix.json @@ -1,12 +1,12 @@ [ { - "k8sVersion": "1.31", + "k8sVersion": "1.35", "ami": "AL2_x86_64", "instanceType":"t3.medium", "arc": "amd64" }, { - "k8sVersion": "1.31", + "k8sVersion": "1.35", "ami": "AL2_ARM_64", "instanceType":"m6g.large", "arc": "arm64" diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 07fbe3192..8331f987a 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -420,7 +420,7 @@ var testTypeToTestConfig = map[string][]testConfig{ testDir: "./test/liscsi", terraformDir: "terraform/eks/daemon/liscsi", targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - wip: true, + instanceType: "i7i.xlarge", }, { testDir: "./test/otel/standard", diff --git a/terraform/eks/daemon/liscsi/main.tf b/terraform/eks/daemon/liscsi/main.tf index 5d3260212..ee1cf6d6c 100644 --- a/terraform/eks/daemon/liscsi/main.tf +++ b/terraform/eks/daemon/liscsi/main.tf @@ -30,6 +30,7 @@ resource "aws_eks_cluster" "this" { } } +# EKS Node Group resource "aws_eks_node_group" "this" { cluster_name = aws_eks_cluster.this.name node_group_name = "cwagent-liscsi-eks-integ-node-${module.common.testing_id}" @@ -55,6 +56,7 @@ resource "aws_eks_node_group" "this" { ] } +# EKS Node IAM Role resource "aws_iam_role" "node_role" { name = "cwagent-liscsi-eks-Worker-Role-${module.common.testing_id}" @@ -95,14 +97,27 @@ resource "aws_iam_role_policy_attachment" "node_CloudWatchAgentServerPolicy" { } resource "null_resource" "kubectl" { - depends_on = [aws_eks_cluster.this, aws_eks_node_group.this] + depends_on = [ + aws_eks_cluster.this, + aws_eks_node_group.this + ] provisioner "local-exec" { command = <<-EOT ${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name} + ${local.aws_eks} list-clusters --output text + ${local.aws_eks} describe-cluster --name ${aws_eks_cluster.this.name} --output text EOT } } +# LIS CSI addon +resource "aws_eks_addon" "lis_csi_addon" { + depends_on = [aws_eks_node_group.this] + cluster_name = aws_eks_cluster.this.name + addon_name = "aws-ec2-local-instance-store-csi-driver" + configuration_values = jsonencode({ metrics = { enabled = true } }) +} + data "external" "clone_helm_chart" { program = ["bash", "-c", <<-EOT rm -rf ./helm-charts @@ -118,16 +133,22 @@ resource "helm_release" "aws_observability" { namespace = "amazon-cloudwatch" create_namespace = true - set { - name = "clusterName" - value = aws_eks_cluster.this.name - } + set = [ + { + name = "clusterName" + value = aws_eks_cluster.this.name + }, + { + name = "region" + value = var.region + } + ] - set { - name = "region" - value = var.region - } - depends_on = [aws_eks_cluster.this, aws_eks_node_group.this, data.external.clone_helm_chart] + depends_on = [ + aws_eks_cluster.this, + aws_eks_node_group.this, + data.external.clone_helm_chart, + ] } resource "null_resource" "update_image" { @@ -137,41 +158,115 @@ resource "null_resource" "update_image" { } provisioner "local-exec" { command = <<-EOT + echo "Patching CWAgent image to ${var.cwagent_image_repo}:${var.cwagent_image_tag}" kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type='json' -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]' + echo "Waiting for CWAgent DaemonSet rollout..." sleep 10 + kubectl rollout status daemonset/cloudwatch-agent -n amazon-cloudwatch --timeout=300s + echo "CWAgent pods after rollout:" + kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o wide + echo "CWAgent image in use:" + kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o jsonpath='{.items[*].spec.containers[*].image}' + echo "" EOT } } -resource "null_resource" "deploy_mock_lis_csi" { - depends_on = [null_resource.kubectl] +resource "null_resource" "wait_for_lis_csi" { + depends_on = [aws_eks_addon.lis_csi_addon, null_resource.kubectl] provisioner "local-exec" { command = <<-EOT - kubectl apply -f ../../../../test/liscsi/resources/mock-lis-csi.yaml - kubectl rollout status daemonset/mock-nvme-csi-plugin -n kube-system --timeout=300s + echo "Waiting for LIS CSI DaemonSet rollout..." + kubectl rollout status daemonset/ec2-instance-store-plugin -n kube-system --timeout=300s + echo "LIS CSI pods:" + kubectl get pods -n kube-system -l app.kubernetes.io/name=ec2-instance-store-plugin -o wide + echo "StorageClasses:" + kubectl get sc + echo "NVMe devices:" + kubectl get nvmedevices 2>/dev/null || echo "nvmedevices CRD not found" EOT } } +resource "kubernetes_deployment_v1" "lis_csi_io_workload" { + depends_on = [null_resource.wait_for_lis_csi] + metadata { + name = "liscsi-integ-test-io-workload" + namespace = "default" + labels = { + app = "liscsi-integ-test" + } + } + spec { + replicas = 1 + selector { + match_labels = { + app = "liscsi-integ-test" + } + } + template { + metadata { + labels = { + app = "liscsi-integ-test" + } + } + spec { + container { + name = "liscsi-integ-test-writer" + image = "busybox:1.35" + command = ["sh", "-c", "while true; do dd if=/dev/zero of=/data/out.txt bs=1M count=10; sleep 5; done"] + volume_mount { + name = "lis-storage" + mount_path = "/data" + } + } + volume { + name = "lis-storage" + ephemeral { + volume_claim_template { + spec { + access_modes = ["ReadWriteOnce"] + storage_class_name = "ec2-instance-store-sc" + resources { + requests = { + storage = "1Gi" + } + } + } + } + } + } + } + } + } +} + +# Get the single instance ID of the node in the node group data "aws_instances" "eks_node" { - depends_on = [aws_eks_node_group.this] + depends_on = [ + aws_eks_node_group.this + ] filter { name = "tag:eks:nodegroup-name" values = [aws_eks_node_group.this.node_group_name] } } +# Retrieve details of the single instance to get private DNS data "aws_instance" "eks_node_detail" { - depends_on = [data.aws_instances.eks_node] + depends_on = [ + data.aws_instances.eks_node + ] instance_id = data.aws_instances.eks_node.ids[0] } resource "null_resource" "validator" { depends_on = [ aws_eks_node_group.this, + aws_eks_addon.lis_csi_addon, helm_release.aws_observability, null_resource.update_image, - null_resource.deploy_mock_lis_csi, + kubernetes_deployment_v1.lis_csi_io_workload, ] triggers = { @@ -181,12 +276,22 @@ resource "null_resource" "validator" { provisioner "local-exec" { command = <<-EOT echo "Validating CloudWatch Agent with LIS CSI NVME instance store metrics" + echo "=== CWAgent pods ===" + kubectl get pods -n amazon-cloudwatch -o wide + echo "=== CWAgent image ===" + kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/name=cloudwatch-agent -o jsonpath='{.items[*].spec.containers[*].image}' + echo "" + echo "=== LIS CSI pods ===" + kubectl get pods -n kube-system -l app.kubernetes.io/name=ec2-instance-store-plugin -o wide + echo "=== IO workload pods ===" + kubectl get pods -n default -l app=liscsi-integ-test -o wide + echo "=== PVCs ===" + kubectl get pvc -A + echo "=== LIS CSI metrics endpoint ===" + kubectl get svc -n kube-system -l app.kubernetes.io/name=ec2-instance-store-plugin 2>/dev/null || echo "No LIS CSI service found" + echo "=== Running go test ===" cd ../../../.. - go test ${var.test_dir} \ - -eksClusterName=${aws_eks_cluster.this.name} \ - -computeType=EKS -v \ - -eksDeploymentStrategy=DAEMON \ - -instanceId=${data.aws_instance.eks_node_detail.instance_id} + go test ${var.test_dir} -timeout 1h -eksClusterName=${aws_eks_cluster.this.name} -computeType=EKS -v -eksDeploymentStrategy=DAEMON -instanceId=${data.aws_instance.eks_node_detail.instance_id} EOT } } diff --git a/terraform/eks/daemon/liscsi/providers.tf b/terraform/eks/daemon/liscsi/providers.tf index 47638cfd2..ccd805965 100644 --- a/terraform/eks/daemon/liscsi/providers.tf +++ b/terraform/eks/daemon/liscsi/providers.tf @@ -1,34 +1,29 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - helm = { - source = "hashicorp/helm" - version = "~> 2.0" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.0" - } - } -} +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT provider "aws" { region = var.region } -provider "helm" { - kubernetes { - host = aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.this.token - } -} - provider "kubernetes" { + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } host = aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority[0].data) + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) token = data.aws_eks_cluster_auth.this.token } + +provider "helm" { + kubernetes = { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + command = "aws" + } + } +} diff --git a/terraform/eks/daemon/liscsi/variables.tf b/terraform/eks/daemon/liscsi/variables.tf index 3dd53219a..e46ecf550 100644 --- a/terraform/eks/daemon/liscsi/variables.tf +++ b/terraform/eks/daemon/liscsi/variables.tf @@ -28,7 +28,7 @@ variable "helm_chart_branch" { variable "k8s_version" { type = string - default = "1.31" + default = "1.35" } variable "ami_type" { @@ -38,5 +38,5 @@ variable "ami_type" { variable "instance_type" { type = string - default = "t3a.medium" + default = "i7i.xlarge" } diff --git a/test/liscsi/liscsi_runner.go b/test/liscsi/liscsi_runner.go index 0ca9cb40f..29cfcc7a5 100644 --- a/test/liscsi/liscsi_runner.go +++ b/test/liscsi/liscsi_runner.go @@ -73,7 +73,7 @@ func (t *LISCSITestRunner) GetAgentConfigFileName() string { } func (t *LISCSITestRunner) GetAgentRunDuration() time.Duration { - return 2 * time.Minute + return 5 * time.Minute } func (t *LISCSITestRunner) GetMeasuredMetrics() []string { diff --git a/test/liscsi/resources/mock-lis-csi.yaml b/test/liscsi/resources/mock-lis-csi.yaml deleted file mode 100644 index 805142774..000000000 --- a/test/liscsi/resources/mock-lis-csi.yaml +++ /dev/null @@ -1,124 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: mock-lis-csi-code - namespace: kube-system -data: - main.go: | - package main - - import ( - "fmt" - "log" - "net/http" - "os" - ) - - const metricsTemplate = `# HELP aws_ec2_instance_store_csi_ec2_exceeded_iops_seconds_total The total time, in seconds, that the NVMe device exceeded the attached Amazon EC2 instance's maximum IOPS performance - # TYPE aws_ec2_instance_store_csi_ec2_exceeded_iops_seconds_total counter - aws_ec2_instance_store_csi_ec2_exceeded_iops_seconds_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 0 - aws_ec2_instance_store_csi_ec2_exceeded_iops_seconds_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 0 - # HELP aws_ec2_instance_store_csi_ec2_exceeded_tp_seconds_total The total time, in seconds, that the NVMe device exceeded the attached Amazon EC2 instance's maximum throughput performance - # TYPE aws_ec2_instance_store_csi_ec2_exceeded_tp_seconds_total counter - aws_ec2_instance_store_csi_ec2_exceeded_tp_seconds_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 0 - aws_ec2_instance_store_csi_ec2_exceeded_tp_seconds_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 0 - # HELP aws_ec2_instance_store_csi_nvme_collector_errors_total Total number of NVMe collector scrape errors. - # TYPE aws_ec2_instance_store_csi_nvme_collector_errors_total counter - aws_ec2_instance_store_csi_nvme_collector_errors_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal"} 0 - # HELP aws_ec2_instance_store_csi_nvme_collector_scrapes_total Total number of NVMe collector scrapes. - # TYPE aws_ec2_instance_store_csi_nvme_collector_scrapes_total counter - aws_ec2_instance_store_csi_nvme_collector_scrapes_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal"} 2293 - # HELP aws_ec2_instance_store_csi_read_bytes_total The total number of read bytes transferred - # TYPE aws_ec2_instance_store_csi_read_bytes_total counter - aws_ec2_instance_store_csi_read_bytes_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 1.928654848e+09 - aws_ec2_instance_store_csi_read_bytes_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 1.928654848e+09 - # HELP aws_ec2_instance_store_csi_read_ops_total The total number of completed read operations - # TYPE aws_ec2_instance_store_csi_read_ops_total counter - aws_ec2_instance_store_csi_read_ops_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 836193 - aws_ec2_instance_store_csi_read_ops_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 836193 - # HELP aws_ec2_instance_store_csi_read_seconds_total The total time spent on read operations in seconds - # TYPE aws_ec2_instance_store_csi_read_seconds_total counter - aws_ec2_instance_store_csi_read_seconds_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 21.77302 - aws_ec2_instance_store_csi_read_seconds_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 21.712962 - # HELP aws_ec2_instance_store_csi_volume_queue_length The number of read and write operations waiting to be completed - # TYPE aws_ec2_instance_store_csi_volume_queue_length gauge - aws_ec2_instance_store_csi_volume_queue_length{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 1 - aws_ec2_instance_store_csi_volume_queue_length{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 1 - # HELP aws_ec2_instance_store_csi_write_bytes_total The total number of write bytes transferred - # TYPE aws_ec2_instance_store_csi_write_bytes_total counter - aws_ec2_instance_store_csi_write_bytes_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 0 - aws_ec2_instance_store_csi_write_bytes_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 0 - # HELP aws_ec2_instance_store_csi_write_ops_total The total number of completed write operations - # TYPE aws_ec2_instance_store_csi_write_ops_total counter - aws_ec2_instance_store_csi_write_ops_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 0 - aws_ec2_instance_store_csi_write_ops_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 0 - # HELP aws_ec2_instance_store_csi_write_seconds_total The total time spent on write operations in seconds - # TYPE aws_ec2_instance_store_csi_write_seconds_total counter - aws_ec2_instance_store_csi_write_seconds_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme1n1"} 0 - aws_ec2_instance_store_csi_write_seconds_total{instance_id="ip-192-168-31-5.us-west-2.compute.internal",volume_id="/dev/nvme2n1"} 0 - ` - - func metricsHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - fmt.Fprint(w, metricsTemplate) - } - - func main() { - port := os.Getenv("PORT") - if port == "" { - port = "8101" - } - - http.HandleFunc("/metrics", metricsHandler) - log.Printf("Mock LIS CSI server starting on port %s", port) - log.Fatal(http.ListenAndServe(":"+port, nil)) - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: mock-nvme-csi-plugin - namespace: kube-system - labels: - app: nvme-csi-plugin -spec: - selector: - matchLabels: - app: nvme-csi-plugin - template: - metadata: - labels: - app: nvme-csi-plugin - spec: - containers: - - name: mock-lis-csi - image: golang:1.21-alpine - command: ["sh", "-c", "cd /app && go run main.go"] - ports: - - containerPort: 8101 - name: metrics - env: - - name: PORT - value: "8101" - volumeMounts: - - name: code - mountPath: /app - volumes: - - name: code - configMap: - name: mock-lis-csi-code ---- -apiVersion: v1 -kind: Service -metadata: - name: instance-store-csi-metrics - namespace: kube-system - labels: - app: nvme-csi-plugin -spec: - selector: - app: nvme-csi-plugin - ports: - - port: 8101 - targetPort: 8101 - name: metrics