From c9cf9fa2d819b9c42642cae9fe45405791aed871 Mon Sep 17 00:00:00 2001 From: Pooja Reddy Nathala Date: Tue, 12 May 2026 18:04:19 +0000 Subject: [PATCH] Add OTEL LIS CSI MetricsV2 integration tests Add integration tests for Local Instance Store CSI driver metrics validation in the OTEL container insights pipeline. Validates all 13 metrics (9 volume-scoped counters/gauge, 2 latency histograms, 2 collector-internal counters) with proper attribute assertions. - Add test/otel/lis_csi/ test suite following EBS CSI v2 pattern - Add terraform/eks/daemon/otel-lis-csi/ infrastructure - Add otel/lis_csi entry to test case generator - Split metrics into volume-scoped and collector-internal slices for correct volume_id/instance_id assertion scoping --- generator/test_case_generator.go | 7 + terraform/eks/daemon/otel-lis-csi/main.tf | 319 ++++++++++++++++++ .../eks/daemon/otel-lis-csi/providers.tf | 28 ++ .../eks/daemon/otel-lis-csi/variables.tf | 42 +++ test/otel/lis_csi/k8s_helpers_test.go | 163 +++++++++ test/otel/lis_csi/lis_csi_test.go | 142 ++++++++ test/otel/lis_csi/metrics_test.go | 46 +++ test/otel/lis_csi/setup_test.go | 106 ++++++ util/otelmetrics/source_registry.go | 1 + 9 files changed, 854 insertions(+) create mode 100644 terraform/eks/daemon/otel-lis-csi/main.tf create mode 100644 terraform/eks/daemon/otel-lis-csi/providers.tf create mode 100644 terraform/eks/daemon/otel-lis-csi/variables.tf create mode 100644 test/otel/lis_csi/k8s_helpers_test.go create mode 100644 test/otel/lis_csi/lis_csi_test.go create mode 100644 test/otel/lis_csi/metrics_test.go create mode 100644 test/otel/lis_csi/setup_test.go diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index e39d9ad14..3c8f5b22d 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -452,6 +452,13 @@ var testTypeToTestConfig = map[string][]testConfig{ instanceType: "g4dn.xlarge", ami: "AL2023_x86_64_NVIDIA", }, + { + testDir: "./test/otel/lis_csi", + terraformDir: "terraform/eks/daemon/otel-lis-csi", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "i7i.xlarge", + ami: "AL2023_x86_64_STANDARD", + }, { testDir: "./test/otel/multi_efa", terraformDir: "terraform/eks/daemon/otel-multi-efa", diff --git a/terraform/eks/daemon/otel-lis-csi/main.tf b/terraform/eks/daemon/otel-lis-csi/main.tf new file mode 100644 index 000000000..755096683 --- /dev/null +++ b/terraform/eks/daemon/otel-lis-csi/main.tf @@ -0,0 +1,319 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +module "common" { + source = "../../../common" + cwagent_image_repo = var.cwagent_image_repo + cwagent_image_tag = var.cwagent_image_tag +} + +module "basic_components" { + source = "../../../basic_components" + region = var.region +} + +locals { + aws_eks = "aws eks --region ${var.region}" +} + +resource "aws_eks_cluster" "this" { + name = "cwagent-eks-integ-${module.common.testing_id}" + role_arn = module.basic_components.role_arn + version = var.k8s_version + vpc_config { + subnet_ids = module.basic_components.public_subnet_ids + security_group_ids = [module.basic_components.security_group] + } +} + +# EKS Node Group +resource "aws_eks_node_group" "this" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "cwagent-otel-liscsi-integ-node-${module.common.testing_id}" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = module.basic_components.public_subnet_ids + + scaling_config { + desired_size = 1 + max_size = 1 + min_size = 1 + } + + ami_type = var.ami_type + capacity_type = "ON_DEMAND" + disk_size = 20 + instance_types = [var.instance_type] + + depends_on = [ + aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy, + aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy, + ] +} + +# EKS Node IAM Role +resource "aws_iam_role" "node_role" { + name = "cwagent-otel-liscsi-eks-Worker-Role-${module.common.testing_id}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEKSWorkerNodePolicy" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEKS_CNI_Policy" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role_policy_attachment" "node_AmazonEC2ContainerRegistryReadOnly" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + role = aws_iam_role.node_role.name +} + +resource "aws_iam_role_policy_attachment" "node_CloudWatchAgentServerPolicy" { + policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" + role = aws_iam_role.node_role.name +} + +# Pod Identity IAM Role +resource "aws_iam_role" "pod_identity_role" { + name = "cwagent-otel-liscsi-pod-identity-${module.common.testing_id}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "pods.eks.amazonaws.com" } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) +} + +resource "aws_iam_role_policy_attachment" "pod_identity_CloudWatchAgentServerPolicy" { + policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" + role = aws_iam_role.pod_identity_role.name +} + +# --- EKS Addon: Pod Identity agent --- + +resource "aws_eks_addon" "pod_identity_agent" { + depends_on = [aws_eks_node_group.this] + cluster_name = aws_eks_cluster.this.name + addon_name = "eks-pod-identity-agent" +} + +# --- Update kubeconfig --- + +resource "null_resource" "kubectl" { + depends_on = [aws_eks_cluster.this, aws_eks_node_group.this] + provisioner "local-exec" { + command = "${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name}" + } +} + +# --- LIS CSI addon --- + +resource "aws_eks_addon" "lis_csi_addon" { + depends_on = [aws_eks_node_group.this] + cluster_name = aws_eks_cluster.this.name + addon_name = "aws-ec2-local-instance-store-csi-driver" + configuration_values = jsonencode({ metrics = { enabled = true } }) +} + +resource "null_resource" "wait_for_lis_csi" { + depends_on = [aws_eks_addon.lis_csi_addon, null_resource.kubectl] + provisioner "local-exec" { + command = <<-EOT + echo "Waiting for LIS CSI DaemonSet rollout..." + kubectl rollout status daemonset/ec2-instance-store-plugin -n kube-system --timeout=300s + echo "LIS CSI pods:" + kubectl get pods -n kube-system -l app.kubernetes.io/name=ec2-instance-store-plugin -o wide + echo "StorageClasses:" + kubectl get sc + EOT + } +} + +# --- Helm chart install --- + +data "external" "clone_helm_chart" { + program = ["bash", "-c", <<-EOT + rm -rf ./helm-charts + git clone -b ${var.helm_chart_branch} https://github.com/aws-observability/helm-charts.git ./helm-charts + echo '{"status":"ready"}' + EOT + ] +} + +resource "helm_release" "aws_observability" { + name = "amazon-cloudwatch-observability" + chart = "./helm-charts/charts/amazon-cloudwatch-observability" + namespace = "amazon-cloudwatch" + create_namespace = true + + set = [ + { name = "clusterName", value = aws_eks_cluster.this.name }, + { name = "region", value = var.region } + ] + + depends_on = [ + aws_eks_addon.pod_identity_agent, + null_resource.kubectl, + data.external.clone_helm_chart, + ] +} + +# --- Pod Identity association (after Helm creates the service account) --- + +resource "aws_eks_pod_identity_association" "cloudwatch_agent" { + depends_on = [helm_release.aws_observability] + cluster_name = aws_eks_cluster.this.name + namespace = "amazon-cloudwatch" + service_account = "cloudwatch-agent" + role_arn = aws_iam_role.pod_identity_role.arn +} + +# --- Patch agent image --- + +resource "null_resource" "update_image" { + depends_on = [helm_release.aws_observability, null_resource.kubectl] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + sleep 30 + kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type='json' \ + -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]' + sleep 10 + EOT + } +} + +# --- Restart pods to pick up Pod Identity + new image --- + +resource "null_resource" "restart_pods" { + depends_on = [aws_eks_pod_identity_association.cloudwatch_agent, null_resource.update_image] + triggers = { timestamp = timestamp() } + provisioner "local-exec" { + command = <<-EOT + kubectl -n amazon-cloudwatch rollout restart daemonset/cloudwatch-agent + kubectl -n amazon-cloudwatch rollout status daemonset/cloudwatch-agent --timeout=120s + EOT + } +} + +# --- Test workload: nginx --- + +resource "kubernetes_deployment_v1" "nginx_test" { + depends_on = [aws_eks_node_group.this] + metadata { + name = "nginx-test" + namespace = "default" + } + spec { + replicas = 1 + selector { match_labels = { app = "nginx-test" } } + template { + metadata { labels = { app = "nginx-test" } } + spec { + container { + name = "nginx" + image = "public.ecr.aws/nginx/nginx:latest" + port { container_port = 80 } + } + } + } + } +} + +# --- LIS CSI IO workload --- + +resource "kubernetes_deployment_v1" "lis_csi_io_workload" { + depends_on = [null_resource.wait_for_lis_csi] + metadata { + name = "liscsi-integ-test-io-workload" + namespace = "default" + labels = { + app = "liscsi-integ-test" + } + } + spec { + replicas = 1 + selector { + match_labels = { + app = "liscsi-integ-test" + } + } + template { + metadata { + labels = { + app = "liscsi-integ-test" + } + } + spec { + container { + name = "liscsi-integ-test-writer" + image = "busybox:1.35" + command = ["sh", "-c", "while true; do dd if=/dev/zero of=/data/out.txt bs=1M count=10; sleep 5; done"] + volume_mount { + name = "lis-storage" + mount_path = "/data" + } + } + volume { + name = "lis-storage" + ephemeral { + volume_claim_template { + spec { + access_modes = ["ReadWriteOnce"] + storage_class_name = "ec2-instance-store-sc" + resources { + requests = { + storage = "1Gi" + } + } + } + } + } + } + } + } + } +} + +# --- Test runner --- + +resource "null_resource" "validator" { + depends_on = [ + null_resource.restart_pods, + null_resource.wait_for_lis_csi, + kubernetes_deployment_v1.nginx_test, + kubernetes_deployment_v1.lis_csi_io_workload, + ] + + triggers = { always_run = timestamp() } + + provisioner "local-exec" { + command = <<-EOT + echo "Running OTEL LIS CSI integration tests" + cd ../../../.. + + echo "Waiting 3 minutes for metrics to propagate..." + sleep 180 + + go test -tags integration -timeout 1h -v ${var.test_dir} \ + -eksClusterName=${aws_eks_cluster.this.name} \ + -computeType=EKS \ + -eksDeploymentStrategy=DAEMON \ + -region=${var.region} + EOT + } +} diff --git a/terraform/eks/daemon/otel-lis-csi/providers.tf b/terraform/eks/daemon/otel-lis-csi/providers.tf new file mode 100644 index 000000000..168abb217 --- /dev/null +++ b/terraform/eks/daemon/otel-lis-csi/providers.tf @@ -0,0 +1,28 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +provider "aws" { + region = var.region +} + +provider "kubernetes" { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } +} + +provider "helm" { + kubernetes = { + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } + } +} diff --git a/terraform/eks/daemon/otel-lis-csi/variables.tf b/terraform/eks/daemon/otel-lis-csi/variables.tf new file mode 100644 index 000000000..ef535b384 --- /dev/null +++ b/terraform/eks/daemon/otel-lis-csi/variables.tf @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT + +variable "region" { + type = string + default = "us-west-2" +} + +variable "test_dir" { + type = string + default = "./test/otel/lis_csi" +} + +variable "cwagent_image_repo" { + type = string + default = "public.ecr.aws/cloudwatch-agent/cloudwatch-agent" +} + +variable "cwagent_image_tag" { + type = string + default = "latest" +} + +variable "helm_chart_branch" { + type = string + default = "main" +} + +variable "k8s_version" { + type = string + default = "1.35" +} + +variable "ami_type" { + type = string + default = "AL2023_x86_64_STANDARD" +} + +variable "instance_type" { + type = string + default = "i7i.xlarge" +} diff --git a/test/otel/lis_csi/k8s_helpers_test.go b/test/otel/lis_csi/k8s_helpers_test.go new file mode 100644 index 000000000..658073b2f --- /dev/null +++ b/test/otel/lis_csi/k8s_helpers_test.go @@ -0,0 +1,163 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package lis_csi + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// k8sGroundTruth holds node and pod data fetched from the Kubernetes API. +type k8sGroundTruth struct { + nodes map[string]corev1.Node // keyed by metadata.name + pods map[string]corev1.Pod // keyed by "namespace/name" +} + +var ( + groundTruth *k8sGroundTruth + groundTruthOnce sync.Once + groundTruthErr error +) + +// getGroundTruth returns the shared ground truth, initializing it on first call. +func getGroundTruth(t *testing.T) *k8sGroundTruth { + t.Helper() + groundTruthOnce.Do(func() { + groundTruth, groundTruthErr = buildGroundTruth() + }) + if groundTruthErr != nil { + t.Fatalf("failed to build K8s ground truth: %v", groundTruthErr) + } + return groundTruth +} + +func buildGroundTruth() (*k8sGroundTruth, error) { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + kubeconfig = filepath.Join(os.Getenv("HOME"), ".kube", "config") + } + + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, fmt.Errorf("building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("creating K8s clientset: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + nodeList, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("listing nodes: %w", err) + } + if len(nodeList.Items) == 0 { + return nil, fmt.Errorf("K8s API returned 0 nodes") + } + + podList, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("listing pods: %w", err) + } + + gt := &k8sGroundTruth{ + nodes: make(map[string]corev1.Node, len(nodeList.Items)), + pods: make(map[string]corev1.Pod, len(podList.Items)), + } + for _, n := range nodeList.Items { + gt.nodes[n.Name] = n + } + for _, p := range podList.Items { + gt.pods[p.Namespace+"/"+p.Name] = p + } + return gt, nil +} + +// imageTagFromPod finds a pod by namespace+label and returns the image tag of its first container. +func imageTagFromPod(t *testing.T, gt *k8sGroundTruth, namespace, labelKey, labelValue string) string { + t.Helper() + for _, p := range gt.pods { + if p.Namespace != namespace { + continue + } + if v, ok := p.Labels[labelKey]; ok && v == labelValue { + if len(p.Spec.Containers) > 0 { + image := p.Spec.Containers[0].Image + if idx := strings.LastIndex(image, ":"); idx != -1 { + return image[idx+1:] + } + } + } + } + t.Logf("WARNING: no pod found with %s=%s in %s", labelKey, labelValue, namespace) + return "" +} + +// k8sServerVersion returns the Kubernetes API server version string. +func k8sServerVersion(t *testing.T) string { + t.Helper() + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + kubeconfig = filepath.Join(os.Getenv("HOME"), ".kube", "config") + } + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + t.Fatalf("building kubeconfig: %v", err) + } + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + t.Fatalf("creating K8s clientset: %v", err) + } + sv, err := clientset.Discovery().ServerVersion() + if err != nil { + t.Fatalf("getting K8s server version: %v", err) + } + return sv.GitVersion +} + +// lookupPod finds a pod by name and optional namespace. Uses the +// namespace/name key for O(1) lookup. Otherwise falls back to linear scan. +func (gt *k8sGroundTruth) lookupPod(podName, namespace string) (corev1.Pod, bool) { + if namespace != "" { + p, ok := gt.pods[namespace+"/"+podName] + return p, ok + } + for _, p := range gt.pods { + if p.Name == podName { + return p, true + } + } + return corev1.Pod{}, false +} + +// parseInstanceIDFromProviderID extracts the EC2 instance ID from a +// Kubernetes node's spec.providerID. +// Format: "aws:///us-east-1a/i-0abc123def456" → "i-0abc123def456" +func parseInstanceIDFromProviderID(providerID string) (string, error) { + parts := strings.Split(providerID, "/") + if len(parts) == 0 { + return "", fmt.Errorf("empty provider ID") + } + instanceID := parts[len(parts)-1] + if !strings.HasPrefix(instanceID, "i-") { + return "", fmt.Errorf("provider ID %q: last segment %q does not start with 'i-'", providerID, instanceID) + } + return instanceID, nil +} diff --git a/test/otel/lis_csi/lis_csi_test.go b/test/otel/lis_csi/lis_csi_test.go new file mode 100644 index 000000000..c1ff32a15 --- /dev/null +++ b/test/otel/lis_csi/lis_csi_test.go @@ -0,0 +1,142 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package lis_csi + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLISCSIPrerequisites(t *testing.T) { + t.Parallel() + gt := getGroundTruth(t) + + t.Run("pod_running", func(t *testing.T) { + t.Parallel() + found := false + for _, p := range gt.pods { + p := p + if p.Labels["app"] == "liscsi-integ-test" && p.Status.Phase == "Running" { + found = true + break + } + } + require.True(t, found, "no running liscsi-integ-test pod found") + }) +} + +func TestLISCSIInstrumentation(t *testing.T) { + t.Parallel() + for _, metricName := range lisCsiMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err, "querying %s", metricName) + require.NotEmpty(t, results, "%s not available", metricName) + for _, r := range results { + r := r + name, ok := r.Labels.Instrumentation["@name"] + require.True(t, ok, "%s missing @instrumentation.@name", metricName) + require.Equal(t, scopePrometheus, name, "%s instrumentation name", metricName) + } + }) + } +} + +func TestLISCSIVolumeId(t *testing.T) { + t.Parallel() + for _, metricName := range lisCsiVolumeMetricNames() { + metricName := metricName + t.Run(metricName+"/volume_id", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err) + require.NotEmpty(t, results) + for _, r := range results { + r := r + volID, ok := r.Labels.Resource["volume_id"] + require.True(t, ok, "%s missing @resource.volume_id", metricName) + require.True(t, strings.HasPrefix(volID, "pvc-"), "%s volume_id should start with 'pvc-', got '%s'", metricName, volID) + } + }) + } +} + +func TestLISCSINoHwLabels(t *testing.T) { + t.Parallel() + hwAttrs := []string{"hw.type", "hw.vendor", "hw.id"} + for _, metricName := range lisCsiMetricNames() { + metricName := metricName + t.Run(metricName, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err) + require.NotEmpty(t, results) + for _, r := range results { + r := r + for _, attr := range hwAttrs { + attr := attr + _, has := r.Labels.Resource[attr] + require.True(t, !has, "%s should not have @resource.%s", metricName, attr) + } + } + }) + } +} + +func TestLISCSIInstanceId(t *testing.T) { + t.Parallel() + for _, metricName := range lisCsiVolumeMetricNames() { + metricName := metricName + + t.Run(metricName+"/resource_present", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err) + require.NotEmpty(t, results) + for _, r := range results { + r := r + instID, ok := r.Labels.Resource["instance_id"] + require.True(t, ok, "%s missing @resource.instance_id", metricName) + require.True(t, strings.HasPrefix(instID, "i-"), "%s instance_id should start with 'i-', got '%s'", metricName, instID) + } + }) + + t.Run(metricName+"/datapoint_absent", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err) + require.NotEmpty(t, results) + for _, r := range results { + r := r + _, has := r.Labels.Datapoint["instance_id"] + require.True(t, !has, "%s should not have datapoint instance_id", metricName) + } + }) + + t.Run(metricName+"/volume_id_datapoint_absent", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + results, err := queryCache.Get(ctx, metricName) + require.NoError(t, err) + require.NotEmpty(t, results) + for _, r := range results { + r := r + _, has := r.Labels.Datapoint["volume_id"] + require.True(t, !has, "%s should not have datapoint volume_id", metricName) + } + }) + } +} diff --git a/test/otel/lis_csi/metrics_test.go b/test/otel/lis_csi/metrics_test.go new file mode 100644 index 000000000..add1f70b9 --- /dev/null +++ b/test/otel/lis_csi/metrics_test.go @@ -0,0 +1,46 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package lis_csi + +import "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" + +// lisCsiVolumeMetrics are per-volume metrics that carry instance_id and volume_id. +var lisCsiVolumeMetrics = []otelmetrics.MetricDefinition{ + {Name: "aws_ec2_instance_store_csi_read_ops_total", MetricType: "counter", Scope: otelmetrics.ScopePod}, + {Name: "aws_ec2_instance_store_csi_write_ops_total", MetricType: "counter", Scope: otelmetrics.ScopePod}, + {Name: "aws_ec2_instance_store_csi_read_bytes_total", MetricType: "counter", Scope: otelmetrics.ScopePod, Unit: "By"}, + {Name: "aws_ec2_instance_store_csi_write_bytes_total", MetricType: "counter", Scope: otelmetrics.ScopePod, Unit: "By"}, + {Name: "aws_ec2_instance_store_csi_read_seconds_total", MetricType: "counter", Scope: otelmetrics.ScopePod}, + {Name: "aws_ec2_instance_store_csi_write_seconds_total", MetricType: "counter", Scope: otelmetrics.ScopePod}, + {Name: "aws_ec2_instance_store_csi_ec2_exceeded_iops_seconds_total", MetricType: "counter", Scope: otelmetrics.ScopePod}, + {Name: "aws_ec2_instance_store_csi_ec2_exceeded_tp_seconds_total", MetricType: "counter", Scope: otelmetrics.ScopePod}, + {Name: "aws_ec2_instance_store_csi_volume_queue_length", MetricType: "gauge", Scope: otelmetrics.ScopePod}, + {Name: "aws_ec2_instance_store_csi_read_io_latency_seconds", MetricType: "histogram", Scope: otelmetrics.ScopePod}, + {Name: "aws_ec2_instance_store_csi_write_io_latency_seconds", MetricType: "histogram", Scope: otelmetrics.ScopePod}, +} + +// lisCsiCollectorMetrics are collector-internal metrics that do not carry volume_id/instance_id. +var lisCsiCollectorMetrics = []otelmetrics.MetricDefinition{ + {Name: "aws_ec2_instance_store_csi_nvme_collector_errors_total", MetricType: "counter", Scope: otelmetrics.ScopeNode}, + {Name: "aws_ec2_instance_store_csi_nvme_collector_scrapes_total", MetricType: "counter", Scope: otelmetrics.ScopeNode}, +} + +func lisCsiMetricNames() []string { + all := append(lisCsiVolumeMetrics, lisCsiCollectorMetrics...) + names := make([]string, len(all)) + for i, d := range all { + names[i] = d.Name + } + return names +} + +func lisCsiVolumeMetricNames() []string { + names := make([]string, len(lisCsiVolumeMetrics)) + for i, d := range lisCsiVolumeMetrics { + names[i] = d.Name + } + return names +} diff --git a/test/otel/lis_csi/setup_test.go b/test/otel/lis_csi/setup_test.go new file mode 100644 index 000000000..3493936cb --- /dev/null +++ b/test/otel/lis_csi/setup_test.go @@ -0,0 +1,106 @@ +//go:build integration + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package lis_csi + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sts" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +var ( + cfg otelmetrics.TestConfig + client *otelmetrics.OtelMetricsClient + queryCache *otelmetrics.QueryCache +) + +const scopePrometheus = "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" + +var clusterHostTypes = []string{"i7i.xlarge"} + +func TestMain(m *testing.M) { + environment.RegisterEnvironmentMetaDataFlags() + flag.Parse() + + env := environment.GetEnvironmentMetaData() + region := env.Region + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + fmt.Fprintf(os.Stderr, "Region not set\n") + os.Exit(1) + } + + clusterName := env.EKSClusterName + if clusterName == "" { + clusterName = os.Getenv("CLUSTER_NAME") + } + if clusterName == "" { + fmt.Fprintf(os.Stderr, "Cluster name not set\n") + os.Exit(1) + } + + ctx := context.Background() + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + fmt.Fprintf(os.Stderr, "AWS config error: %v\n", err) + os.Exit(1) + } + + stsClient := sts.NewFromConfig(awsCfg) + identity, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + fmt.Fprintf(os.Stderr, "STS GetCallerIdentity error: %v\n", err) + os.Exit(1) + } + + cfg = otelmetrics.TestConfig{ + Region: region, + Endpoint: fmt.Sprintf("https://monitoring.%s.amazonaws.com", region), + Timeout: 30 * time.Second, + MaxRetries: 3, + ClusterName: clusterName, + AccountID: *identity.Account, + SigningService: "monitoring", + } + + client, err = otelmetrics.NewClient(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Client error: %v\n", err) + os.Exit(1) + } + + hostMappings := []otelmetrics.SourceHostMapping{ + {Source: otelmetrics.SourceNodeExporter, HostTypes: clusterHostTypes}, + {Source: otelmetrics.SourceCadvisor, HostTypes: clusterHostTypes}, + {Source: otelmetrics.SourceKubeletstats, HostTypes: clusterHostTypes}, + {Source: otelmetrics.SourceControlPlane, HostTypes: nil}, + {Source: otelmetrics.SourceKubeStateMetrics, HostTypes: nil}, + {Source: otelmetrics.SourceKSMNodeScoped, HostTypes: nil}, + {Source: otelmetrics.SourceLISCSI, HostTypes: nil}, + } + + registry := otelmetrics.NewSourceRegistry(clusterHostTypes, hostMappings, + otelmetrics.SourceMapping{Source: otelmetrics.SourceLISCSI, Metrics: append(lisCsiVolumeMetrics, lisCsiCollectorMetrics...)}, + ) + + queryCache = otelmetrics.NewQueryCache(client, cfg.ClusterName, + otelmetrics.WithHostTypes(clusterHostTypes), + otelmetrics.WithSourceRegistry(registry), + ) + + os.Exit(m.Run()) +} diff --git a/util/otelmetrics/source_registry.go b/util/otelmetrics/source_registry.go index 74d7b047f..4e91a4177 100644 --- a/util/otelmetrics/source_registry.go +++ b/util/otelmetrics/source_registry.go @@ -14,6 +14,7 @@ const ( SourceNeuron SourceEFA SourceEBSCSI + SourceLISCSI SourceControlPlane SourceKubeStateMetrics SourceKSMNodeScoped