From 36408838aeb791c8731dece68e3115e6f5565e7e Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Thu, 25 Jun 2026 12:15:25 +0100 Subject: [PATCH 01/14] feat(otel): add Karpenter integration test and Terraform setup --- terraform/eks/daemon/otel/main.tf | 223 +++++++++++++++++- terraform/eks/daemon/otel/variables.tf | 5 + .../integrations/karpenter/karpenter_test.go | 117 +++++++++ .../integrations/karpenter/metrics_test.go | 63 +++++ .../otel/integrations/karpenter/setup_test.go | 101 ++++++++ util/otelmetrics/source_registry.go | 1 + 6 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 test/otel/integrations/karpenter/karpenter_test.go create mode 100644 test/otel/integrations/karpenter/metrics_test.go create mode 100644 test/otel/integrations/karpenter/setup_test.go diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index 22b3d5de2..5a1097cd6 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -13,11 +13,14 @@ module "basic_components" { } locals { - aws_eks = "aws eks --region ${var.region}" + aws_eks = "aws eks --region ${var.region}" + cluster_name = "cwagent-eks-integ-${module.common.testing_id}" } +data "aws_partition" "current" {} + resource "aws_eks_cluster" "this" { - name = "cwagent-eks-integ-${module.common.testing_id}" + name = local.cluster_name role_arn = module.basic_components.role_arn version = var.k8s_version vpc_config { @@ -110,6 +113,130 @@ resource "aws_eks_addon" "pod_identity_agent" { addon_name = "eks-pod-identity-agent" } +# ----------------------------------------------------------------------------- +# Karpenter Controller IAM Role (Pod Identity) +# ----------------------------------------------------------------------------- + +resource "aws_iam_role" "karpenter_controller" { + name = "cwagent-otel-karpenter-${module.common.testing_id}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "pods.eks.amazonaws.com" } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) +} + +resource "aws_iam_role_policy" "karpenter_controller" { + name = "KarpenterControllerPolicy" + role = aws_iam_role.karpenter_controller.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "EC2Permissions" + Effect = "Allow" + Action = [ + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplate", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:RunInstances", + "ec2:TerminateInstances", + ] + Resource = "*" + }, + { + Sid = "PassRoleToEC2" + Effect = "Allow" + Action = "iam:PassRole" + Resource = aws_iam_role.node_role.arn + }, + { + Sid = "EKSPermissions" + Effect = "Allow" + Action = "eks:DescribeCluster" + Resource = aws_eks_cluster.this.arn + }, + { + Sid = "SSMGetParameter" + Effect = "Allow" + Action = "ssm:GetParameter" + Resource = "arn:${data.aws_partition.current.partition}:ssm:${var.region}::parameter/aws/service/*" + }, + { + Sid = "PricingPermissions" + Effect = "Allow" + Action = "pricing:GetProducts" + Resource = "*" + }, + { + Sid = "SQSPermissions" + Effect = "Allow" + Action = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + ] + Resource = aws_sqs_queue.karpenter_interruption.arn + }, + ] + }) +} + +# Karpenter Interruption Queue (SQS) +resource "aws_sqs_queue" "karpenter_interruption" { + name = "cwagent-otel-karpenter-${module.common.testing_id}" + message_retention_seconds = 300 + sqs_managed_sse_enabled = true +} + +resource "aws_sqs_queue_policy" "karpenter_interruption" { + queue_url = aws_sqs_queue.karpenter_interruption.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = ["events.amazonaws.com", "sqs.amazonaws.com"] } + Action = "sqs:SendMessage" + Resource = aws_sqs_queue.karpenter_interruption.arn + }] + }) +} + +# Karpenter Pod Identity Association +resource "aws_eks_pod_identity_association" "karpenter_controller" { + depends_on = [aws_eks_addon.pod_identity_agent] + cluster_name = aws_eks_cluster.this.name + namespace = "kube-system" + service_account = "karpenter" + role_arn = aws_iam_role.karpenter_controller.arn +} + +# Tag subnets and security group for Karpenter discovery +resource "aws_ec2_tag" "subnet_karpenter" { + count = length(module.basic_components.public_subnet_ids) + resource_id = module.basic_components.public_subnet_ids[count.index] + key = "karpenter.sh/discovery" + value = aws_eks_cluster.this.name +} + +resource "aws_ec2_tag" "sg_karpenter" { + resource_id = module.basic_components.security_group + key = "karpenter.sh/discovery" + value = aws_eks_cluster.this.name +} + # --- Update kubeconfig --- resource "null_resource" "kubectl" { @@ -119,6 +246,88 @@ resource "null_resource" "kubectl" { } } +# --- Karpenter Helm install --- + +resource "helm_release" "karpenter" { + name = "karpenter" + repository = "oci://public.ecr.aws/karpenter" + chart = "karpenter" + version = var.karpenter_version + namespace = "kube-system" + create_namespace = false + + set = [ + { name = "settings.clusterName", value = aws_eks_cluster.this.name }, + { name = "settings.clusterEndpoint", value = aws_eks_cluster.this.endpoint }, + { name = "settings.interruptionQueue", value = aws_sqs_queue.karpenter_interruption.name }, + { name = "controller.resources.requests.cpu", value = "100m" }, + { name = "controller.resources.requests.memory", value = "256Mi" }, + ] + + depends_on = [ + aws_eks_addon.pod_identity_agent, + aws_eks_pod_identity_association.karpenter_controller, + null_resource.kubectl, + ] +} + +# --- Karpenter NodePool + EC2NodeClass (minimal, for metric generation) --- + +resource "null_resource" "karpenter_nodepool" { + depends_on = [helm_release.karpenter] + provisioner "local-exec" { + command = <<-EOT + cat <<'EOF' | kubectl apply -f - + apiVersion: karpenter.sh/v1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["t"] + - key: "karpenter.k8s.aws/instance-size" + operator: In + values: ["medium", "large"] + - key: "kubernetes.io/arch" + operator: In + values: ["amd64"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["on-demand"] + limits: + cpu: "4" + memory: "8Gi" + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 60s + --- + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + metadata: + name: default + spec: + role: "${aws_iam_role.node_role.name}" + amiSelectorTerms: + - alias: al2023@latest + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${aws_eks_cluster.this.name}" + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${aws_eks_cluster.this.name}" + EOF + EOT + } +} + # --- Helm chart install --- data "external" "clone_helm_chart" { @@ -140,12 +349,14 @@ resource "helm_release" "aws_observability" { { name = "clusterName", value = aws_eks_cluster.this.name }, { name = "region", value = var.region }, { name = "otelContainerInsights.enabled", value = "true" }, + { name = "otelContainerInsights.metrics.karpenter.enabled", value = "true" }, ] depends_on = [ aws_eks_addon.pod_identity_agent, null_resource.kubectl, data.external.clone_helm_chart, + helm_release.karpenter, ] } @@ -351,6 +562,7 @@ resource "null_resource" "validator" { depends_on = [ null_resource.restart_pods, kubernetes_deployment_v1.nginx_test, + null_resource.karpenter_nodepool, ] triggers = { always_run = timestamp() } @@ -368,6 +580,13 @@ resource "null_resource" "validator" { -computeType=EKS \ -eksDeploymentStrategy=DAEMON \ -region=${var.region} + + echo "Running OTEL integrations tests (Karpenter)..." + go test -tags integration -timeout 1h -v ./test/otel/integrations/karpenter/... \ + -eksClusterName=${aws_eks_cluster.this.name} \ + -computeType=EKS \ + -eksDeploymentStrategy=DAEMON \ + -region=${var.region} EOT } } diff --git a/terraform/eks/daemon/otel/variables.tf b/terraform/eks/daemon/otel/variables.tf index 9149462f2..7ed580859 100644 --- a/terraform/eks/daemon/otel/variables.tf +++ b/terraform/eks/daemon/otel/variables.tf @@ -40,3 +40,8 @@ variable "instance_type" { type = string default = "t3.medium" } + +variable "karpenter_version" { + type = string + default = "1.4.0" +} diff --git a/test/otel/integrations/karpenter/karpenter_test.go b/test/otel/integrations/karpenter/karpenter_test.go new file mode 100644 index 000000000..b3e86b52e --- /dev/null +++ b/test/otel/integrations/karpenter/karpenter_test.go @@ -0,0 +1,117 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package karpenter + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestKarpenterMetricsExist verifies that each expected Karpenter metric is present. +func TestKarpenterMetricsExist(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available (is Karpenter installed?)", metricName) + }) + } +} + +// TestKarpenterInstrumentation verifies instrumentation source for all Karpenter metrics. +func TestKarpenterInstrumentation(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + r := r + name, ok := r.Labels.Instrumentation["@name"] + require.True(t, ok, "%s missing @instrumentation.@name", metricName) + require.Equal(t, scopeKarpenter, name, "%s instrumentation name", metricName) + } + }) + } +} + +// TestKarpenterClusterName verifies the cluster name resource attribute is set correctly. +func TestKarpenterClusterName(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + r := r + cluster, ok := r.Labels.Resource["k8s.cluster.name"] + require.True(t, ok, "%s missing @resource.k8s.cluster.name", metricName) + require.Equal(t, cfg.ClusterName, cluster, "%s cluster name mismatch", metricName) + } + }) + } +} + +// TestKarpenterExpectedLabels verifies that expected datapoint labels are present. +func TestKarpenterExpectedLabels(t *testing.T) { + t.Parallel() + for _, md := range karpenterMetrics { + md := md + if len(md.ExpectedLabels) == 0 { + continue + } + t.Run(md.Name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, md.Name) + require.NoError(t, err, "querying %s", md.Name) + require.NotEmpty(t, results, "%s not available", md.Name) + for _, r := range results { + r := r + for _, label := range md.ExpectedLabels { + label := label + _, ok := r.Labels.Datapoint[label] + require.True(t, ok, "%s missing expected label '%s'", md.Name, label) + } + } + }) + } +} + +// TestKarpenterNamespace verifies that Karpenter metrics have a namespace resource attribute. +func TestKarpenterNamespace(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + r := r + ns, ok := r.Labels.Resource["k8s.namespace.name"] + require.True(t, ok, "%s missing @resource.k8s.namespace.name", metricName) + require.NotEmpty(t, ns, "%s empty k8s.namespace.name", metricName) + } + }) + } +} diff --git a/test/otel/integrations/karpenter/metrics_test.go b/test/otel/integrations/karpenter/metrics_test.go new file mode 100644 index 000000000..f332f8f74 --- /dev/null +++ b/test/otel/integrations/karpenter/metrics_test.go @@ -0,0 +1,63 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package karpenter + +import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" + +// karpenterMetrics defines the Karpenter controller metrics expected in CloudWatch. +// These are Prometheus metrics scraped from the Karpenter controller pods and +// forwarded by the CloudWatch Agent's Karpenter integration pipeline. +var karpenterMetrics = []otelmetrics.MetricDefinition{ + // Build info + {Name: "karpenter_build_info", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + + // Pod scheduling metrics + {Name: "karpenter_pods_state", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"phase"}}, + {Name: "karpenter_pods_bound_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, Unit: "s"}, + + // Scheduler metrics + {Name: "karpenter_scheduler_ignored_pods_count", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_scheduler_scheduling_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, Unit: "s"}, + + // Node resource metrics + {Name: "karpenter_nodes_allocatable", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_total_daemon_requests", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_total_daemon_limits", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_total_pod_requests", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_total_pod_limits", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_system_overhead", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_current_lifetime_seconds", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, Unit: "s"}, + + // Cluster state metrics + {Name: "karpenter_cluster_state_node_count", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_cluster_state_synced", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_cluster_state_unsynced_time_seconds", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, Unit: "s"}, + {Name: "karpenter_cluster_utilization_percent", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + + // NodePool metrics + {Name: "karpenter_nodepools_limit", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodepools_usage", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + + // Cloudprovider metrics + {Name: "karpenter_cloudprovider_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"method"}, Unit: "s"}, + {Name: "karpenter_cloudprovider_instance_type_cpu_cores", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_cloudprovider_instance_type_memory_bytes", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, Unit: "By"}, + {Name: "karpenter_cloudprovider_instance_type_offering_available", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_cloudprovider_instance_type_offering_price_estimate", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + + // Disruption metrics + {Name: "karpenter_voluntary_disruption_eligible_nodes", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"reason"}}, + {Name: "karpenter_voluntary_disruption_consolidation_timeouts_total", MetricType: "counter", Scope: otelmetrics.ScopeCluster, Unit: "1"}, + {Name: "karpenter_voluntary_disruption_decision_evaluation_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, Unit: "s"}, +} + +func karpenterMetricNames() []string { + names := make([]string, len(karpenterMetrics)) + for i, d := range karpenterMetrics { + names[i] = d.Name + } + return names +} diff --git a/test/otel/integrations/karpenter/setup_test.go b/test/otel/integrations/karpenter/setup_test.go new file mode 100644 index 000000000..c3f437853 --- /dev/null +++ b/test/otel/integrations/karpenter/setup_test.go @@ -0,0 +1,101 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package karpenter + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sts" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +var ( + cfg otelmetrics.TestConfig + client *otelmetrics.OtelMetricsClient + queryCache *otelmetrics.QueryCache +) + +// Instrumentation scope name constant. +const scopeKarpenter = "github.com/aws/karpenter" + +// Instance types in the Karpenter cluster. +var clusterHostTypes = []string{"t3.medium"} + +func TestMain(m *testing.M) { + environment.RegisterEnvironmentMetaDataFlags() + flag.Parse() + env := environment.GetEnvironmentMetaData() + + region := env.Region + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + fmt.Fprintf(os.Stderr, "Region not set\n") + os.Exit(1) + } + + clusterName := env.EKSClusterName + if clusterName == "" { + clusterName = os.Getenv("CLUSTER_NAME") + } + if clusterName == "" { + fmt.Fprintf(os.Stderr, "Cluster name not set\n") + os.Exit(1) + } + + ctx := context.Background() + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + fmt.Fprintf(os.Stderr, "AWS config error: %v\n", err) + os.Exit(1) + } + stsClient := sts.NewFromConfig(awsCfg) + identity, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + fmt.Fprintf(os.Stderr, "STS GetCallerIdentity error: %v\n", err) + os.Exit(1) + } + + cfg = otelmetrics.TestConfig{ + Region: region, + Endpoint: fmt.Sprintf("https://monitoring.%s.amazonaws.com", region), + Timeout: 30 * time.Second, + MaxRetries: 3, + ClusterName: clusterName, + AccountID: *identity.Account, + SigningService: "monitoring", + } + + client, err = otelmetrics.NewClient(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Client error: %v\n", err) + os.Exit(1) + } + + hostMappings := []otelmetrics.SourceHostMapping{ + {Source: otelmetrics.SourceKarpenter, HostTypes: nil}, + } + + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, + otelmetrics.SourceMapping{Source: otelmetrics.SourceKarpenter, Metrics: karpenterMetrics}, + ) + + queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithHostTypes(clusterHostTypes), + otelmetrics.WithSourceRegistry(registry), + ) + + os.Exit(m.Run()) +} diff --git a/util/otelmetrics/source_registry.go b/util/otelmetrics/source_registry.go index 4e91a4177..075793c37 100644 --- a/util/otelmetrics/source_registry.go +++ b/util/otelmetrics/source_registry.go @@ -18,6 +18,7 @@ const ( SourceControlPlane SourceKubeStateMetrics SourceKSMNodeScoped + SourceKarpenter ) // SourceMapping pairs a MetricSource with its metric definitions. From eb630c05f4e5b584b661c909557d93cbd2567f0e Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Thu, 25 Jun 2026 13:38:40 +0100 Subject: [PATCH 02/14] revert locals refactor, keep original cluster name inline --- terraform/eks/daemon/otel/main.tf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index 5a1097cd6..e2132779a 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -13,14 +13,13 @@ module "basic_components" { } locals { - aws_eks = "aws eks --region ${var.region}" - cluster_name = "cwagent-eks-integ-${module.common.testing_id}" + aws_eks = "aws eks --region ${var.region}" } data "aws_partition" "current" {} resource "aws_eks_cluster" "this" { - name = local.cluster_name + name = "cwagent-eks-integ-${module.common.testing_id}" role_arn = module.basic_components.role_arn version = var.k8s_version vpc_config { From 7d1c3b64ab75b9471fc6092d164420f1154e26bd Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Thu, 25 Jun 2026 14:28:31 +0100 Subject: [PATCH 03/14] removed data "aws_partition" "current" from main.tf --- terraform/eks/daemon/otel/main.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index e2132779a..7849d2599 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -16,8 +16,6 @@ locals { aws_eks = "aws eks --region ${var.region}" } -data "aws_partition" "current" {} - resource "aws_eks_cluster" "this" { name = "cwagent-eks-integ-${module.common.testing_id}" role_arn = module.basic_components.role_arn @@ -171,7 +169,7 @@ resource "aws_iam_role_policy" "karpenter_controller" { Sid = "SSMGetParameter" Effect = "Allow" Action = "ssm:GetParameter" - Resource = "arn:${data.aws_partition.current.partition}:ssm:${var.region}::parameter/aws/service/*" + Resource = "arn:aws:ssm:${var.region}::parameter/aws/service/*" }, { Sid = "PricingPermissions" From 28cf6b5ebbf364c9224c7a54f6895d7a1b25f0a6 Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Thu, 25 Jun 2026 17:25:32 +0100 Subject: [PATCH 04/14] test: added instrumentation consistency check --- .../integrations/karpenter/karpenter_test.go | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/otel/integrations/karpenter/karpenter_test.go b/test/otel/integrations/karpenter/karpenter_test.go index b3e86b52e..0483c34f5 100644 --- a/test/otel/integrations/karpenter/karpenter_test.go +++ b/test/otel/integrations/karpenter/karpenter_test.go @@ -115,3 +115,26 @@ func TestKarpenterNamespace(t *testing.T) { }) } } + +// TestKarpenterInstrumentationConsistent verifies all data points for a metric +// report the same instrumentation scope (no mixed sources). +func TestKarpenterInstrumentationConsistent(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + names := make(map[string]struct{}) + for _, r := range results { + if n, ok := r.Labels.Instrumentation["@name"]; ok { + names[n] = struct{}{} + } + } + require.Equal(t, 1, len(names), "%s has %d distinct instrumentation names", metricName, len(names)) + }) + } +} From cf9a4f4acd02b896741740d6ca8424b20f9d536b Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Fri, 26 Jun 2026 13:31:33 +0100 Subject: [PATCH 05/14] edited comment --- terraform/eks/daemon/otel/main.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index 7849d2599..54c48b7aa 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -110,9 +110,7 @@ resource "aws_eks_addon" "pod_identity_agent" { addon_name = "eks-pod-identity-agent" } -# ----------------------------------------------------------------------------- -# Karpenter Controller IAM Role (Pod Identity) -# ----------------------------------------------------------------------------- +# --- Karpenter Controller IAM Role (Pod Identity) --- resource "aws_iam_role" "karpenter_controller" { name = "cwagent-otel-karpenter-${module.common.testing_id}" From ec8908712c9699508492530c29929f05ec954b32 Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Fri, 26 Jun 2026 16:21:17 +0100 Subject: [PATCH 06/14] refactor: consolidate resource attribute tests into single test --- .../integrations/karpenter/karpenter_test.go | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/test/otel/integrations/karpenter/karpenter_test.go b/test/otel/integrations/karpenter/karpenter_test.go index 0483c34f5..ceecc8f15 100644 --- a/test/otel/integrations/karpenter/karpenter_test.go +++ b/test/otel/integrations/karpenter/karpenter_test.go @@ -48,8 +48,9 @@ func TestKarpenterInstrumentation(t *testing.T) { } } -// TestKarpenterClusterName verifies the cluster name resource attribute is set correctly. -func TestKarpenterClusterName(t *testing.T) { +// TestKarpenterInstrumentationConsistent verifies all data points for a metric +// report the same instrumentation scope (no mixed sources). +func TestKarpenterInstrumentationConsistent(t *testing.T) { t.Parallel() for _, metricName := range karpenterMetricNames() { metricName := metricName @@ -59,12 +60,13 @@ func TestKarpenterClusterName(t *testing.T) { results, err := queryCache.Get(ctx, metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) + names := make(map[string]struct{}) for _, r := range results { - r := r - cluster, ok := r.Labels.Resource["k8s.cluster.name"] - require.True(t, ok, "%s missing @resource.k8s.cluster.name", metricName) - require.Equal(t, cfg.ClusterName, cluster, "%s cluster name mismatch", metricName) + if n, ok := r.Labels.Instrumentation["@name"]; ok { + names[n] = struct{}{} + } } + require.Equal(t, 1, len(names), "%s has %d distinct instrumentation names", metricName, len(names)) }) } } @@ -95,31 +97,19 @@ func TestKarpenterExpectedLabels(t *testing.T) { } } -// TestKarpenterNamespace verifies that Karpenter metrics have a namespace resource attribute. -func TestKarpenterNamespace(t *testing.T) { +// TestKarpenterResourceAttributes verifies K8s and cloud resource attributes are enriched +// correctly, including cluster name value validation. +func TestKarpenterResourceAttributes(t *testing.T) { t.Parallel() - for _, metricName := range karpenterMetricNames() { - metricName := metricName - t.Run(metricName, func(t *testing.T) { - t.Parallel() - ctx := context.Background() - results, err := queryCache.Get(ctx, metricName) - require.NoError(t, err, "querying %s", metricName) - require.NotEmpty(t, results, "%s not available", metricName) - for _, r := range results { - r := r - ns, ok := r.Labels.Resource["k8s.namespace.name"] - require.True(t, ok, "%s missing @resource.k8s.namespace.name", metricName) - require.NotEmpty(t, ns, "%s empty k8s.namespace.name", metricName) - } - }) + requiredAttrs := []string{ + "k8s.pod.name", + "k8s.deployment.name", + "k8s.namespace.name", + "k8s.cluster.name", + "cloud.provider", + "cloud.region", + "cloud.platform", } -} - -// TestKarpenterInstrumentationConsistent verifies all data points for a metric -// report the same instrumentation scope (no mixed sources). -func TestKarpenterInstrumentationConsistent(t *testing.T) { - t.Parallel() for _, metricName := range karpenterMetricNames() { metricName := metricName t.Run(metricName, func(t *testing.T) { @@ -128,13 +118,17 @@ func TestKarpenterInstrumentationConsistent(t *testing.T) { results, err := queryCache.Get(ctx, metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) - names := make(map[string]struct{}) for _, r := range results { - if n, ok := r.Labels.Instrumentation["@name"]; ok { - names[n] = struct{}{} + r := r + for _, attr := range requiredAttrs { + v, ok := r.Labels.Resource[attr] + require.True(t, ok, "%s missing @resource.%s", metricName, attr) + require.NotEmpty(t, v, "%s empty @resource.%s", metricName, attr) } + // Validate cluster name matches the expected value + require.Equal(t, cfg.ClusterName, r.Labels.Resource["k8s.cluster.name"], + "%s cluster name mismatch", metricName) } - require.Equal(t, 1, len(names), "%s has %d distinct instrumentation names", metricName, len(names)) }) } } From db96765905d7fd9afa243d6ac118d0d48dc24c96 Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Thu, 2 Jul 2026 15:14:09 +0100 Subject: [PATCH 07/14] test(karpenter): Added value assertions for cloud and K8s resource attributes --- .../integrations/karpenter/karpenter_test.go | 263 ++++++++++++++++-- 1 file changed, 237 insertions(+), 26 deletions(-) diff --git a/test/otel/integrations/karpenter/karpenter_test.go b/test/otel/integrations/karpenter/karpenter_test.go index ceecc8f15..6410ff0a8 100644 --- a/test/otel/integrations/karpenter/karpenter_test.go +++ b/test/otel/integrations/karpenter/karpenter_test.go @@ -7,12 +7,22 @@ package karpenter import ( "context" + "fmt" + "strings" "testing" "github.com/stretchr/testify/require" ) -// TestKarpenterMetricsExist verifies that each expected Karpenter metric is present. +// =========================================================================== +// Metric Existence Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterMetricsExist — verify each expected metric is present in +// CloudWatch. +// --------------------------------------------------------------------------- + func TestKarpenterMetricsExist(t *testing.T) { t.Parallel() for _, metricName := range karpenterMetricNames() { @@ -27,7 +37,15 @@ func TestKarpenterMetricsExist(t *testing.T) { } } -// TestKarpenterInstrumentation verifies instrumentation source for all Karpenter metrics. +// =========================================================================== +// Instrumentation Scope Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterInstrumentation — verify instrumentation scope name for all +// Karpenter metrics. +// --------------------------------------------------------------------------- + func TestKarpenterInstrumentation(t *testing.T) { t.Parallel() for _, metricName := range karpenterMetricNames() { @@ -48,8 +66,11 @@ func TestKarpenterInstrumentation(t *testing.T) { } } -// TestKarpenterInstrumentationConsistent verifies all data points for a metric -// report the same instrumentation scope (no mixed sources). +// --------------------------------------------------------------------------- +// TestKarpenterInstrumentationConsistent — verify all data points for a +// metric report the same instrumentation scope (no mixed sources). +// --------------------------------------------------------------------------- + func TestKarpenterInstrumentationConsistent(t *testing.T) { t.Parallel() for _, metricName := range karpenterMetricNames() { @@ -71,7 +92,15 @@ func TestKarpenterInstrumentationConsistent(t *testing.T) { } } -// TestKarpenterExpectedLabels verifies that expected datapoint labels are present. +// =========================================================================== +// Datapoint Label Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterExpectedLabels — verify expected datapoint labels are present +// on metrics that declare them. +// --------------------------------------------------------------------------- + func TestKarpenterExpectedLabels(t *testing.T) { t.Parallel() for _, md := range karpenterMetrics { @@ -97,37 +126,219 @@ func TestKarpenterExpectedLabels(t *testing.T) { } } -// TestKarpenterResourceAttributes verifies K8s and cloud resource attributes are enriched -// correctly, including cluster name value validation. -func TestKarpenterResourceAttributes(t *testing.T) { +// =========================================================================== +// K8s Resource Attribute Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterClusterIdentity — k8s.cluster.name must match the configured +// cluster. +// --------------------------------------------------------------------------- + +func TestKarpenterClusterIdentity(t *testing.T) { t.Parallel() - requiredAttrs := []string{ - "k8s.pod.name", - "k8s.deployment.name", - "k8s.namespace.name", - "k8s.cluster.name", - "cloud.provider", - "cloud.region", - "cloud.platform", + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + clusterName, ok := r.Labels.Resource["k8s.cluster.name"] + require.True(t, ok, "%s missing @resource.k8s.cluster.name", metricName) + require.Equal(t, cfg.ClusterName, clusterName, "%s k8s.cluster.name", metricName) + } + }) } +} + +// --------------------------------------------------------------------------- +// TestKarpenterNamespace — all Karpenter metrics must originate from the +// kube-system namespace. +// --------------------------------------------------------------------------- + +func TestKarpenterNamespace(t *testing.T) { + t.Parallel() for _, metricName := range karpenterMetricNames() { metricName := metricName t.Run(metricName, func(t *testing.T) { t.Parallel() - ctx := context.Background() - results, err := queryCache.Get(ctx, metricName) + results, err := queryCache.Get(context.Background(), metricName) require.NoError(t, err, "querying %s", metricName) require.NotEmpty(t, results, "%s not available", metricName) for _, r := range results { - r := r - for _, attr := range requiredAttrs { - v, ok := r.Labels.Resource[attr] - require.True(t, ok, "%s missing @resource.%s", metricName, attr) - require.NotEmpty(t, v, "%s empty @resource.%s", metricName, attr) + ns, ok := r.Labels.Resource["k8s.namespace.name"] + require.True(t, ok, "%s missing @resource.k8s.namespace.name", metricName) + require.Equal(t, "kube-system", ns, "%s k8s.namespace.name", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterDeploymentName — k8s.deployment.name must be "karpenter". +// --------------------------------------------------------------------------- + +func TestKarpenterDeploymentName(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + deploy, ok := r.Labels.Resource["k8s.deployment.name"] + require.True(t, ok, "%s missing @resource.k8s.deployment.name", metricName) + require.Equal(t, "karpenter", deploy, "%s k8s.deployment.name", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterPodName — k8s.pod.name must start with "karpenter-". +// --------------------------------------------------------------------------- + +func TestKarpenterPodName(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + podName, ok := r.Labels.Resource["k8s.pod.name"] + require.True(t, ok, "%s missing @resource.k8s.pod.name", metricName) + require.True(t, strings.HasPrefix(podName, "karpenter-"), + "%s k8s.pod.name should start with 'karpenter-', got %q", metricName, podName) + } + }) + } +} + +// =========================================================================== +// Cloud Resource Attribute Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterCloudProvider — cloud.provider must be "aws". +// --------------------------------------------------------------------------- + +func TestKarpenterCloudProvider(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + provider, ok := r.Labels.Resource["cloud.provider"] + require.True(t, ok, "%s missing @resource.cloud.provider", metricName) + require.Equal(t, "aws", provider, "%s cloud.provider", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterCloudPlatform — cloud.platform must be "aws_eks". +// --------------------------------------------------------------------------- + +func TestKarpenterCloudPlatform(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + platform, ok := r.Labels.Resource["cloud.platform"] + require.True(t, ok, "%s missing @resource.cloud.platform", metricName) + require.Equal(t, "aws_eks", platform, "%s cloud.platform", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterCloudRegion — cloud.region must match the configured region. +// --------------------------------------------------------------------------- + +func TestKarpenterCloudRegion(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + region, ok := r.Labels.Resource["cloud.region"] + require.True(t, ok, "%s missing @resource.cloud.region", metricName) + require.Equal(t, cfg.Region, region, "%s cloud.region", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterCloudAccountID — cloud.account.id must match the test account. +// --------------------------------------------------------------------------- + +func TestKarpenterCloudAccountID(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + acctID, ok := r.Labels.Resource["cloud.account.id"] + if ok { + require.Equal(t, cfg.AccountID, acctID, "%s cloud.account.id", metricName) + } + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterCloudResourceID — cloud.resource_id must be a valid EKS +// cluster ARN. +// --------------------------------------------------------------------------- + +func TestKarpenterCloudResourceID(t *testing.T) { + t.Parallel() + expectedARNPrefix := fmt.Sprintf("arn:aws:eks:%s:", cfg.Region) + expectedARNSuffix := fmt.Sprintf(":cluster/%s", cfg.ClusterName) + + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + arn, ok := r.Labels.Resource["cloud.resource_id"] + if ok { + require.True(t, strings.HasPrefix(arn, expectedARNPrefix), + "%s cloud.resource_id should start with %q, got %q", metricName, expectedARNPrefix, arn) + require.True(t, strings.HasSuffix(arn, expectedARNSuffix), + "%s cloud.resource_id should end with %q, got %q", metricName, expectedARNSuffix, arn) } - // Validate cluster name matches the expected value - require.Equal(t, cfg.ClusterName, r.Labels.Resource["k8s.cluster.name"], - "%s cluster name mismatch", metricName) } }) } From a1009f483bc6f504ebb8b323eff7362fb18452a4 Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Mon, 6 Jul 2026 14:52:54 +0100 Subject: [PATCH 08/14] fix : renamed flag name in main.tf --- terraform/eks/daemon/otel/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index 54c48b7aa..c4dad1401 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -344,7 +344,7 @@ resource "helm_release" "aws_observability" { { name = "clusterName", value = aws_eks_cluster.this.name }, { name = "region", value = var.region }, { name = "otelContainerInsights.enabled", value = "true" }, - { name = "otelContainerInsights.metrics.karpenter.enabled", value = "true" }, + { name = "otelContainerInsights.integrations.karpenter.enabled", value = "true" }, ] depends_on = [ From 15652d2bbbfdda85b0645357fea43f8cf444647e Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Wed, 8 Jul 2026 13:12:12 +0100 Subject: [PATCH 09/14] fix: update karpenter to 1.13.0, remove SQS queue --- terraform/eks/daemon/otel/main.tf | 31 -------------------------- terraform/eks/daemon/otel/variables.tf | 2 +- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index c4dad1401..e49ecb518 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -175,40 +175,10 @@ resource "aws_iam_role_policy" "karpenter_controller" { Action = "pricing:GetProducts" Resource = "*" }, - { - Sid = "SQSPermissions" - Effect = "Allow" - Action = [ - "sqs:DeleteMessage", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage", - ] - Resource = aws_sqs_queue.karpenter_interruption.arn - }, ] }) } -# Karpenter Interruption Queue (SQS) -resource "aws_sqs_queue" "karpenter_interruption" { - name = "cwagent-otel-karpenter-${module.common.testing_id}" - message_retention_seconds = 300 - sqs_managed_sse_enabled = true -} - -resource "aws_sqs_queue_policy" "karpenter_interruption" { - queue_url = aws_sqs_queue.karpenter_interruption.id - policy = jsonencode({ - Version = "2012-10-17" - Statement = [{ - Effect = "Allow" - Principal = { Service = ["events.amazonaws.com", "sqs.amazonaws.com"] } - Action = "sqs:SendMessage" - Resource = aws_sqs_queue.karpenter_interruption.arn - }] - }) -} - # Karpenter Pod Identity Association resource "aws_eks_pod_identity_association" "karpenter_controller" { depends_on = [aws_eks_addon.pod_identity_agent] @@ -254,7 +224,6 @@ resource "helm_release" "karpenter" { set = [ { name = "settings.clusterName", value = aws_eks_cluster.this.name }, { name = "settings.clusterEndpoint", value = aws_eks_cluster.this.endpoint }, - { name = "settings.interruptionQueue", value = aws_sqs_queue.karpenter_interruption.name }, { name = "controller.resources.requests.cpu", value = "100m" }, { name = "controller.resources.requests.memory", value = "256Mi" }, ] diff --git a/terraform/eks/daemon/otel/variables.tf b/terraform/eks/daemon/otel/variables.tf index 7ed580859..095e2e6ce 100644 --- a/terraform/eks/daemon/otel/variables.tf +++ b/terraform/eks/daemon/otel/variables.tf @@ -43,5 +43,5 @@ variable "instance_type" { variable "karpenter_version" { type = string - default = "1.4.0" + default = "1.13.0" } From 260dda526bb2fb9e8d860d7433b3ba3064d01f6f Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Wed, 8 Jul 2026 14:31:47 +0100 Subject: [PATCH 10/14] ci: temporarily point helm chart to fork for testing --- terraform/eks/daemon/otel/main.tf | 2 +- terraform/eks/daemon/otel/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index e49ecb518..9b5283603 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -297,7 +297,7 @@ resource "null_resource" "karpenter_nodepool" { data "external" "clone_helm_chart" { program = ["bash", "-c", <<-EOT rm -rf ./helm-charts - git clone -b ${var.helm_chart_branch} https://github.com/aws-observability/helm-charts.git ./helm-charts + git clone -b ${var.helm_chart_branch} https://github.com/vaishnavi-30-beep/helm-charts.git ./helm-charts echo '{"status":"ready"}' EOT ] diff --git a/terraform/eks/daemon/otel/variables.tf b/terraform/eks/daemon/otel/variables.tf index 095e2e6ce..33b56a0af 100644 --- a/terraform/eks/daemon/otel/variables.tf +++ b/terraform/eks/daemon/otel/variables.tf @@ -23,7 +23,7 @@ variable "cwagent_image_tag" { variable "helm_chart_branch" { type = string - default = "main" + default = "feature/karpenter-integration" } variable "k8s_version" { From 556c4fc2f0dc5d5bd22bfdf8a40c8bb663d8ea13 Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Wed, 8 Jul 2026 19:01:19 +0100 Subject: [PATCH 11/14] fix: remove scheduler metrics that require pod scheduling activity --- test/otel/integrations/karpenter/metrics_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/otel/integrations/karpenter/metrics_test.go b/test/otel/integrations/karpenter/metrics_test.go index f332f8f74..77639fbce 100644 --- a/test/otel/integrations/karpenter/metrics_test.go +++ b/test/otel/integrations/karpenter/metrics_test.go @@ -18,10 +18,6 @@ var karpenterMetrics = []otelmetrics.MetricDefinition{ {Name: "karpenter_pods_state", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"phase"}}, {Name: "karpenter_pods_bound_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, Unit: "s"}, - // Scheduler metrics - {Name: "karpenter_scheduler_ignored_pods_count", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, - {Name: "karpenter_scheduler_scheduling_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, Unit: "s"}, - // Node resource metrics {Name: "karpenter_nodes_allocatable", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, {Name: "karpenter_nodes_total_daemon_requests", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, From 2270369ba1e5a07571fe6cde2b1318a110c8ed0c Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Wed, 8 Jul 2026 19:16:11 +0100 Subject: [PATCH 12/14] fix: rename helm_chart_branch to avoid CI override, remove scheduler metrics --- terraform/eks/daemon/otel/main.tf | 2 +- terraform/eks/daemon/otel/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index 9b5283603..90c532215 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -297,7 +297,7 @@ resource "null_resource" "karpenter_nodepool" { data "external" "clone_helm_chart" { program = ["bash", "-c", <<-EOT rm -rf ./helm-charts - git clone -b ${var.helm_chart_branch} https://github.com/vaishnavi-30-beep/helm-charts.git ./helm-charts + git clone -b ${var.karpenter_helm_chart_branch} https://github.com/vaishnavi-30-beep/helm-charts.git ./helm-charts echo '{"status":"ready"}' EOT ] diff --git a/terraform/eks/daemon/otel/variables.tf b/terraform/eks/daemon/otel/variables.tf index 33b56a0af..a9ed23e6c 100644 --- a/terraform/eks/daemon/otel/variables.tf +++ b/terraform/eks/daemon/otel/variables.tf @@ -21,7 +21,7 @@ variable "cwagent_image_tag" { default = "latest" } -variable "helm_chart_branch" { +variable "karpenter_helm_chart_branch" { type = string default = "feature/karpenter-integration" } From 156d1125a367ca52f349870934e4eff1d953137c Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Thu, 9 Jul 2026 00:08:20 +0100 Subject: [PATCH 13/14] fix: rename to karpenter_chart_ref to avoid CI grep override --- terraform/eks/daemon/otel/main.tf | 2 +- terraform/eks/daemon/otel/variables.tf | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index 90c532215..69a2ffa49 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -297,7 +297,7 @@ resource "null_resource" "karpenter_nodepool" { data "external" "clone_helm_chart" { program = ["bash", "-c", <<-EOT rm -rf ./helm-charts - git clone -b ${var.karpenter_helm_chart_branch} https://github.com/vaishnavi-30-beep/helm-charts.git ./helm-charts + git clone -b ${var.karpenter_chart_ref} https://github.com/vaishnavi-30-beep/helm-charts.git ./helm-charts echo '{"status":"ready"}' EOT ] diff --git a/terraform/eks/daemon/otel/variables.tf b/terraform/eks/daemon/otel/variables.tf index a9ed23e6c..caccb44ba 100644 --- a/terraform/eks/daemon/otel/variables.tf +++ b/terraform/eks/daemon/otel/variables.tf @@ -21,11 +21,17 @@ variable "cwagent_image_tag" { default = "latest" } -variable "karpenter_helm_chart_branch" { +variable "karpenter_chart_ref" { type = string default = "feature/karpenter-integration" } +variable "helm_chart_branch" { + type = string + default = "main" + description = "Passed by CI workflow but unused — karpenter_chart_ref is used instead" +} + variable "k8s_version" { type = string default = "1.35" From abc9d22985a1c04b47ee4a3dcbca648c4d1b6356 Mon Sep 17 00:00:00 2001 From: Vaishnavi Joshi Date: Thu, 9 Jul 2026 11:19:23 +0100 Subject: [PATCH 14/14] revert: remove temporary fork/branch overrides --- terraform/eks/daemon/otel/main.tf | 2 +- terraform/eks/daemon/otel/variables.tf | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index 69a2ffa49..e49ecb518 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -297,7 +297,7 @@ resource "null_resource" "karpenter_nodepool" { data "external" "clone_helm_chart" { program = ["bash", "-c", <<-EOT rm -rf ./helm-charts - git clone -b ${var.karpenter_chart_ref} https://github.com/vaishnavi-30-beep/helm-charts.git ./helm-charts + git clone -b ${var.helm_chart_branch} https://github.com/aws-observability/helm-charts.git ./helm-charts echo '{"status":"ready"}' EOT ] diff --git a/terraform/eks/daemon/otel/variables.tf b/terraform/eks/daemon/otel/variables.tf index caccb44ba..095e2e6ce 100644 --- a/terraform/eks/daemon/otel/variables.tf +++ b/terraform/eks/daemon/otel/variables.tf @@ -21,15 +21,9 @@ variable "cwagent_image_tag" { default = "latest" } -variable "karpenter_chart_ref" { - type = string - default = "feature/karpenter-integration" -} - variable "helm_chart_branch" { type = string default = "main" - description = "Passed by CI workflow but unused — karpenter_chart_ref is used instead" } variable "k8s_version" {