diff --git a/terraform/eks/daemon/otel/main.tf b/terraform/eks/daemon/otel/main.tf index 22b3d5de..e49ecb51 100644 --- a/terraform/eks/daemon/otel/main.tf +++ b/terraform/eks/daemon/otel/main.tf @@ -110,6 +110,98 @@ resource "aws_eks_addon" "pod_identity_agent" { addon_name = "eks-pod-identity-agent" } +# --- Karpenter Controller IAM Role (Pod Identity) --- + +resource "aws_iam_role" "karpenter_controller" { + name = "cwagent-otel-karpenter-${module.common.testing_id}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "pods.eks.amazonaws.com" } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) +} + +resource "aws_iam_role_policy" "karpenter_controller" { + name = "KarpenterControllerPolicy" + role = aws_iam_role.karpenter_controller.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "EC2Permissions" + Effect = "Allow" + Action = [ + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplate", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:RunInstances", + "ec2:TerminateInstances", + ] + Resource = "*" + }, + { + Sid = "PassRoleToEC2" + Effect = "Allow" + Action = "iam:PassRole" + Resource = aws_iam_role.node_role.arn + }, + { + Sid = "EKSPermissions" + Effect = "Allow" + Action = "eks:DescribeCluster" + Resource = aws_eks_cluster.this.arn + }, + { + Sid = "SSMGetParameter" + Effect = "Allow" + Action = "ssm:GetParameter" + Resource = "arn:aws:ssm:${var.region}::parameter/aws/service/*" + }, + { + Sid = "PricingPermissions" + Effect = "Allow" + Action = "pricing:GetProducts" + Resource = "*" + }, + ] + }) +} + +# Karpenter Pod Identity Association +resource "aws_eks_pod_identity_association" "karpenter_controller" { + depends_on = [aws_eks_addon.pod_identity_agent] + cluster_name = aws_eks_cluster.this.name + namespace = "kube-system" + service_account = "karpenter" + role_arn = aws_iam_role.karpenter_controller.arn +} + +# Tag subnets and security group for Karpenter discovery +resource "aws_ec2_tag" "subnet_karpenter" { + count = length(module.basic_components.public_subnet_ids) + resource_id = module.basic_components.public_subnet_ids[count.index] + key = "karpenter.sh/discovery" + value = aws_eks_cluster.this.name +} + +resource "aws_ec2_tag" "sg_karpenter" { + resource_id = module.basic_components.security_group + key = "karpenter.sh/discovery" + value = aws_eks_cluster.this.name +} + # --- Update kubeconfig --- resource "null_resource" "kubectl" { @@ -119,6 +211,87 @@ resource "null_resource" "kubectl" { } } +# --- Karpenter Helm install --- + +resource "helm_release" "karpenter" { + name = "karpenter" + repository = "oci://public.ecr.aws/karpenter" + chart = "karpenter" + version = var.karpenter_version + namespace = "kube-system" + create_namespace = false + + set = [ + { name = "settings.clusterName", value = aws_eks_cluster.this.name }, + { name = "settings.clusterEndpoint", value = aws_eks_cluster.this.endpoint }, + { name = "controller.resources.requests.cpu", value = "100m" }, + { name = "controller.resources.requests.memory", value = "256Mi" }, + ] + + depends_on = [ + aws_eks_addon.pod_identity_agent, + aws_eks_pod_identity_association.karpenter_controller, + null_resource.kubectl, + ] +} + +# --- Karpenter NodePool + EC2NodeClass (minimal, for metric generation) --- + +resource "null_resource" "karpenter_nodepool" { + depends_on = [helm_release.karpenter] + provisioner "local-exec" { + command = <<-EOT + cat <<'EOF' | kubectl apply -f - + apiVersion: karpenter.sh/v1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["t"] + - key: "karpenter.k8s.aws/instance-size" + operator: In + values: ["medium", "large"] + - key: "kubernetes.io/arch" + operator: In + values: ["amd64"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["on-demand"] + limits: + cpu: "4" + memory: "8Gi" + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 60s + --- + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + metadata: + name: default + spec: + role: "${aws_iam_role.node_role.name}" + amiSelectorTerms: + - alias: al2023@latest + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${aws_eks_cluster.this.name}" + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${aws_eks_cluster.this.name}" + EOF + EOT + } +} + # --- Helm chart install --- data "external" "clone_helm_chart" { @@ -140,12 +313,14 @@ resource "helm_release" "aws_observability" { { name = "clusterName", value = aws_eks_cluster.this.name }, { name = "region", value = var.region }, { name = "otelContainerInsights.enabled", value = "true" }, + { name = "otelContainerInsights.integrations.karpenter.enabled", value = "true" }, ] depends_on = [ aws_eks_addon.pod_identity_agent, null_resource.kubectl, data.external.clone_helm_chart, + helm_release.karpenter, ] } @@ -351,6 +526,7 @@ resource "null_resource" "validator" { depends_on = [ null_resource.restart_pods, kubernetes_deployment_v1.nginx_test, + null_resource.karpenter_nodepool, ] triggers = { always_run = timestamp() } @@ -368,6 +544,13 @@ resource "null_resource" "validator" { -computeType=EKS \ -eksDeploymentStrategy=DAEMON \ -region=${var.region} + + echo "Running OTEL integrations tests (Karpenter)..." + go test -tags integration -timeout 1h -v ./test/otel/integrations/karpenter/... \ + -eksClusterName=${aws_eks_cluster.this.name} \ + -computeType=EKS \ + -eksDeploymentStrategy=DAEMON \ + -region=${var.region} EOT } } diff --git a/terraform/eks/daemon/otel/variables.tf b/terraform/eks/daemon/otel/variables.tf index 9149462f..095e2e6c 100644 --- a/terraform/eks/daemon/otel/variables.tf +++ b/terraform/eks/daemon/otel/variables.tf @@ -40,3 +40,8 @@ variable "instance_type" { type = string default = "t3.medium" } + +variable "karpenter_version" { + type = string + default = "1.13.0" +} diff --git a/test/otel/integrations/karpenter/karpenter_test.go b/test/otel/integrations/karpenter/karpenter_test.go new file mode 100644 index 00000000..6410ff0a --- /dev/null +++ b/test/otel/integrations/karpenter/karpenter_test.go @@ -0,0 +1,345 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package karpenter + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// =========================================================================== +// Metric Existence Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterMetricsExist — verify each expected metric is present in +// CloudWatch. +// --------------------------------------------------------------------------- + +func TestKarpenterMetricsExist(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available (is Karpenter installed?)", metricName) + }) + } +} + +// =========================================================================== +// Instrumentation Scope Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterInstrumentation — verify instrumentation scope name for all +// Karpenter metrics. +// --------------------------------------------------------------------------- + +func TestKarpenterInstrumentation(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + r := r + name, ok := r.Labels.Instrumentation["@name"] + require.True(t, ok, "%s missing @instrumentation.@name", metricName) + require.Equal(t, scopeKarpenter, name, "%s instrumentation name", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterInstrumentationConsistent — verify all data points for a +// metric report the same instrumentation scope (no mixed sources). +// --------------------------------------------------------------------------- + +func TestKarpenterInstrumentationConsistent(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + names := make(map[string]struct{}) + for _, r := range results { + if n, ok := r.Labels.Instrumentation["@name"]; ok { + names[n] = struct{}{} + } + } + require.Equal(t, 1, len(names), "%s has %d distinct instrumentation names", metricName, len(names)) + }) + } +} + +// =========================================================================== +// Datapoint Label Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterExpectedLabels — verify expected datapoint labels are present +// on metrics that declare them. +// --------------------------------------------------------------------------- + +func TestKarpenterExpectedLabels(t *testing.T) { + t.Parallel() + for _, md := range karpenterMetrics { + md := md + if len(md.ExpectedLabels) == 0 { + continue + } + t.Run(md.Name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, md.Name) + require.NoError(t, err, "querying %s", md.Name) + require.NotEmpty(t, results, "%s not available", md.Name) + for _, r := range results { + r := r + for _, label := range md.ExpectedLabels { + label := label + _, ok := r.Labels.Datapoint[label] + require.True(t, ok, "%s missing expected label '%s'", md.Name, label) + } + } + }) + } +} + +// =========================================================================== +// K8s Resource Attribute Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterClusterIdentity — k8s.cluster.name must match the configured +// cluster. +// --------------------------------------------------------------------------- + +func TestKarpenterClusterIdentity(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + clusterName, ok := r.Labels.Resource["k8s.cluster.name"] + require.True(t, ok, "%s missing @resource.k8s.cluster.name", metricName) + require.Equal(t, cfg.ClusterName, clusterName, "%s k8s.cluster.name", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterNamespace — all Karpenter metrics must originate from the +// kube-system namespace. +// --------------------------------------------------------------------------- + +func TestKarpenterNamespace(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + ns, ok := r.Labels.Resource["k8s.namespace.name"] + require.True(t, ok, "%s missing @resource.k8s.namespace.name", metricName) + require.Equal(t, "kube-system", ns, "%s k8s.namespace.name", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterDeploymentName — k8s.deployment.name must be "karpenter". +// --------------------------------------------------------------------------- + +func TestKarpenterDeploymentName(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + deploy, ok := r.Labels.Resource["k8s.deployment.name"] + require.True(t, ok, "%s missing @resource.k8s.deployment.name", metricName) + require.Equal(t, "karpenter", deploy, "%s k8s.deployment.name", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterPodName — k8s.pod.name must start with "karpenter-". +// --------------------------------------------------------------------------- + +func TestKarpenterPodName(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + podName, ok := r.Labels.Resource["k8s.pod.name"] + require.True(t, ok, "%s missing @resource.k8s.pod.name", metricName) + require.True(t, strings.HasPrefix(podName, "karpenter-"), + "%s k8s.pod.name should start with 'karpenter-', got %q", metricName, podName) + } + }) + } +} + +// =========================================================================== +// Cloud Resource Attribute Tests +// =========================================================================== + +// --------------------------------------------------------------------------- +// TestKarpenterCloudProvider — cloud.provider must be "aws". +// --------------------------------------------------------------------------- + +func TestKarpenterCloudProvider(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + provider, ok := r.Labels.Resource["cloud.provider"] + require.True(t, ok, "%s missing @resource.cloud.provider", metricName) + require.Equal(t, "aws", provider, "%s cloud.provider", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterCloudPlatform — cloud.platform must be "aws_eks". +// --------------------------------------------------------------------------- + +func TestKarpenterCloudPlatform(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + platform, ok := r.Labels.Resource["cloud.platform"] + require.True(t, ok, "%s missing @resource.cloud.platform", metricName) + require.Equal(t, "aws_eks", platform, "%s cloud.platform", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterCloudRegion — cloud.region must match the configured region. +// --------------------------------------------------------------------------- + +func TestKarpenterCloudRegion(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + region, ok := r.Labels.Resource["cloud.region"] + require.True(t, ok, "%s missing @resource.cloud.region", metricName) + require.Equal(t, cfg.Region, region, "%s cloud.region", metricName) + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterCloudAccountID — cloud.account.id must match the test account. +// --------------------------------------------------------------------------- + +func TestKarpenterCloudAccountID(t *testing.T) { + t.Parallel() + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + acctID, ok := r.Labels.Resource["cloud.account.id"] + if ok { + require.Equal(t, cfg.AccountID, acctID, "%s cloud.account.id", metricName) + } + } + }) + } +} + +// --------------------------------------------------------------------------- +// TestKarpenterCloudResourceID — cloud.resource_id must be a valid EKS +// cluster ARN. +// --------------------------------------------------------------------------- + +func TestKarpenterCloudResourceID(t *testing.T) { + t.Parallel() + expectedARNPrefix := fmt.Sprintf("arn:aws:eks:%s:", cfg.Region) + expectedARNSuffix := fmt.Sprintf(":cluster/%s", cfg.ClusterName) + + for _, metricName := range karpenterMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + results, err := queryCache.Get(context.Background(), metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + arn, ok := r.Labels.Resource["cloud.resource_id"] + if ok { + require.True(t, strings.HasPrefix(arn, expectedARNPrefix), + "%s cloud.resource_id should start with %q, got %q", metricName, expectedARNPrefix, arn) + require.True(t, strings.HasSuffix(arn, expectedARNSuffix), + "%s cloud.resource_id should end with %q, got %q", metricName, expectedARNSuffix, arn) + } + } + }) + } +} diff --git a/test/otel/integrations/karpenter/metrics_test.go b/test/otel/integrations/karpenter/metrics_test.go new file mode 100644 index 00000000..77639fbc --- /dev/null +++ b/test/otel/integrations/karpenter/metrics_test.go @@ -0,0 +1,59 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package karpenter + +import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" + +// karpenterMetrics defines the Karpenter controller metrics expected in CloudWatch. +// These are Prometheus metrics scraped from the Karpenter controller pods and +// forwarded by the CloudWatch Agent's Karpenter integration pipeline. +var karpenterMetrics = []otelmetrics.MetricDefinition{ + // Build info + {Name: "karpenter_build_info", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + + // Pod scheduling metrics + {Name: "karpenter_pods_state", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"phase"}}, + {Name: "karpenter_pods_bound_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, Unit: "s"}, + + // Node resource metrics + {Name: "karpenter_nodes_allocatable", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_total_daemon_requests", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_total_daemon_limits", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_total_pod_requests", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_total_pod_limits", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_system_overhead", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodes_current_lifetime_seconds", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, Unit: "s"}, + + // Cluster state metrics + {Name: "karpenter_cluster_state_node_count", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_cluster_state_synced", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_cluster_state_unsynced_time_seconds", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, Unit: "s"}, + {Name: "karpenter_cluster_utilization_percent", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + + // NodePool metrics + {Name: "karpenter_nodepools_limit", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + {Name: "karpenter_nodepools_usage", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"resource_type"}}, + + // Cloudprovider metrics + {Name: "karpenter_cloudprovider_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"method"}, Unit: "s"}, + {Name: "karpenter_cloudprovider_instance_type_cpu_cores", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_cloudprovider_instance_type_memory_bytes", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, Unit: "By"}, + {Name: "karpenter_cloudprovider_instance_type_offering_available", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + {Name: "karpenter_cloudprovider_instance_type_offering_price_estimate", MetricType: "gauge", Scope: otelmetrics.ScopeCluster}, + + // Disruption metrics + {Name: "karpenter_voluntary_disruption_eligible_nodes", MetricType: "gauge", Scope: otelmetrics.ScopeCluster, ExpectedLabels: []string{"reason"}}, + {Name: "karpenter_voluntary_disruption_consolidation_timeouts_total", MetricType: "counter", Scope: otelmetrics.ScopeCluster, Unit: "1"}, + {Name: "karpenter_voluntary_disruption_decision_evaluation_duration_seconds", MetricType: "histogram", Scope: otelmetrics.ScopeCluster, Unit: "s"}, +} + +func karpenterMetricNames() []string { + names := make([]string, len(karpenterMetrics)) + for i, d := range karpenterMetrics { + names[i] = d.Name + } + return names +} diff --git a/test/otel/integrations/karpenter/setup_test.go b/test/otel/integrations/karpenter/setup_test.go new file mode 100644 index 00000000..c3f43785 --- /dev/null +++ b/test/otel/integrations/karpenter/setup_test.go @@ -0,0 +1,101 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package karpenter + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sts" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +var ( + cfg otelmetrics.TestConfig + client *otelmetrics.OtelMetricsClient + queryCache *otelmetrics.QueryCache +) + +// Instrumentation scope name constant. +const scopeKarpenter = "github.com/aws/karpenter" + +// Instance types in the Karpenter cluster. +var clusterHostTypes = []string{"t3.medium"} + +func TestMain(m *testing.M) { + environment.RegisterEnvironmentMetaDataFlags() + flag.Parse() + env := environment.GetEnvironmentMetaData() + + region := env.Region + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + fmt.Fprintf(os.Stderr, "Region not set\n") + os.Exit(1) + } + + clusterName := env.EKSClusterName + if clusterName == "" { + clusterName = os.Getenv("CLUSTER_NAME") + } + if clusterName == "" { + fmt.Fprintf(os.Stderr, "Cluster name not set\n") + os.Exit(1) + } + + ctx := context.Background() + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + fmt.Fprintf(os.Stderr, "AWS config error: %v\n", err) + os.Exit(1) + } + stsClient := sts.NewFromConfig(awsCfg) + identity, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + fmt.Fprintf(os.Stderr, "STS GetCallerIdentity error: %v\n", err) + os.Exit(1) + } + + cfg = otelmetrics.TestConfig{ + Region: region, + Endpoint: fmt.Sprintf("https://monitoring.%s.amazonaws.com", region), + Timeout: 30 * time.Second, + MaxRetries: 3, + ClusterName: clusterName, + AccountID: *identity.Account, + SigningService: "monitoring", + } + + client, err = otelmetrics.NewClient(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Client error: %v\n", err) + os.Exit(1) + } + + hostMappings := []otelmetrics.SourceHostMapping{ + {Source: otelmetrics.SourceKarpenter, HostTypes: nil}, + } + + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, + otelmetrics.SourceMapping{Source: otelmetrics.SourceKarpenter, Metrics: karpenterMetrics}, + ) + + queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithHostTypes(clusterHostTypes), + otelmetrics.WithSourceRegistry(registry), + ) + + os.Exit(m.Run()) +} diff --git a/util/otelmetrics/source_registry.go b/util/otelmetrics/source_registry.go index 4e91a417..075793c3 100644 --- a/util/otelmetrics/source_registry.go +++ b/util/otelmetrics/source_registry.go @@ -18,6 +18,7 @@ const ( SourceControlPlane SourceKubeStateMetrics SourceKSMNodeScoped + SourceKarpenter ) // SourceMapping pairs a MetricSource with its metric definitions.