From ea78a4603f50d9954419aa3d09d559162426f024 Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Thu, 9 Apr 2026 09:10:31 -0300
Subject: [PATCH 01/10] feat: add ALB metrics publishing to CloudWatch and
 Datadog

Publish ALB rule count and target group count as custom metrics
after each deployment to enable continuous monitoring and alerting
on ALB quota usage.

- Add publish_alb_metrics script supporting CloudWatch and Datadog
- Integrate as post-deploy step in initial, switch_traffic, and finalize workflows
- Add ALB_METRICS_PUBLISH_ENABLED and ALB_METRICS_PUBLISH_TARGET config
- Add .claude/settings.local.json to .gitignore
---
 .gitignore                                   |   5 +-
 k8s/deployment/publish_alb_metrics           | 140 +++++++++++++++++++
 k8s/deployment/workflows/finalize.yaml       |  16 ++-
 k8s/deployment/workflows/initial.yaml        |   3 +
 k8s/deployment/workflows/switch_traffic.yaml |  16 ++-
 k8s/values.yaml                              |   2 +
 6 files changed, 171 insertions(+), 11 deletions(-)
 create mode 100755 k8s/deployment/publish_alb_metrics

diff --git a/.gitignore b/.gitignore
index dc24eb3e..30d0b975 100644
--- a/.gitignore
+++ b/.gitignore
@@ -134,4 +134,7 @@ dist
 .idea
 k8s/output
 np-agent-manifest.yaml
-.minikube_mount_pid
\ No newline at end of file
+.minikube_mount_pid
+
+# Claude Code
+.claude/settings.local.json
\ No newline at end of file
diff --git a/k8s/deployment/publish_alb_metrics b/k8s/deployment/publish_alb_metrics
new file mode 100755
index 00000000..5bff54d2
--- /dev/null
+++ b/k8s/deployment/publish_alb_metrics
@@ -0,0 +1,140 @@
+#!/bin/bash
+# Post-deployment ALB metrics publisher
+# Publishes ALB rule count and target group count as custom metrics
+# to CloudWatch or Datadog for continuous monitoring and alerting.
+
+ALB_RECONCILIATION_ENABLED="${ALB_RECONCILIATION_ENABLED:-false}"
+ALB_METRICS_PUBLISH_ENABLED="${ALB_METRICS_PUBLISH_ENABLED:-false}"
+ALB_METRICS_PUBLISH_TARGET="${ALB_METRICS_PUBLISH_TARGET:-cloudwatch}"
+
+if [ "$ALB_RECONCILIATION_ENABLED" != "true" ] || [ "$ALB_METRICS_PUBLISH_ENABLED" != "true" ]; then
+    echo "ℹ ALB metrics publishing skipped (ALB_RECONCILIATION_ENABLED=$ALB_RECONCILIATION_ENABLED, ALB_METRICS_PUBLISH_ENABLED=$ALB_METRICS_PUBLISH_ENABLED)"
+    return 0
+fi
+
+ALB_NAME=$(echo "$CONTEXT" | jq -r '.alb_name')
+REGION=$(echo "$CONTEXT" | jq -r '.region')
+
+if [ -z "$ALB_NAME" ] || [ "$ALB_NAME" = "null" ]; then
+    echo "⚠ ALB name not found in context, skipping metrics publishing"
+    return 0
+fi
+
+echo "Publishing ALB metrics for [$ALB_NAME] in region [$REGION]..."
+
+# Resolve ALB ARN
+ALB_ARN=$(aws elbv2 describe-load-balancers \
+    --names "$ALB_NAME" \
+    --region "$REGION" \
+    --query 'LoadBalancers[0].LoadBalancerArn' \
+    --output text 2>&1)
+
+if [ $? -ne 0 ] || [ "$ALB_ARN" = "None" ] || [ -z "$ALB_ARN" ]; then
+    echo "⚠ Could not find ALB [$ALB_NAME], skipping metrics publishing"
+    return 0
+fi
+
+# Count rules across all listeners
+TOTAL_RULES=0
+LISTENERS=$(aws elbv2 describe-listeners \
+    --load-balancer-arn "$ALB_ARN" \
+    --region "$REGION" \
+    --output json 2>&1)
+
+if [ $? -ne 0 ]; then
+    echo "⚠ Could not retrieve listeners for ALB [$ALB_NAME], skipping metrics publishing"
+    return 0
+fi
+
+LISTENER_ARNS=$(echo "$LISTENERS" | jq -r '.Listeners[].ListenerArn')
+
+for listener_arn in $LISTENER_ARNS; do
+    RULES=$(aws elbv2 describe-rules \
+        --listener-arn "$listener_arn" \
+        --region "$REGION" \
+        --output json 2>&1)
+
+    if [ $? -eq 0 ]; then
+        LISTENER_RULE_COUNT=$(echo "$RULES" | jq '[.Rules[] | select(.IsDefault != true)] | length')
+        TOTAL_RULES=$((TOTAL_RULES + LISTENER_RULE_COUNT))
+    fi
+done
+
+# Count target groups
+TARGET_GROUPS=$(aws elbv2 describe-target-groups \
+    --load-balancer-arn "$ALB_ARN" \
+    --region "$REGION" \
+    --output json 2>&1)
+
+TG_COUNT=0
+if [ $? -eq 0 ]; then
+    TG_COUNT=$(echo "$TARGET_GROUPS" | jq '.TargetGroups | length')
+fi
+
+echo "  Rules: $TOTAL_RULES"
+echo "  Target Groups: $TG_COUNT"
+
+# Publish metrics based on target
+case "$ALB_METRICS_PUBLISH_TARGET" in
+    cloudwatch)
+        echo "Publishing to CloudWatch (namespace: Nullplatform/ALB)..."
+
+        aws cloudwatch put-metric-data \
+            --namespace "Nullplatform/ALB" \
+            --metric-data "[
+                {\"MetricName\":\"RuleCount\",\"Value\":$TOTAL_RULES,\"Unit\":\"Count\",\"Dimensions\":[{\"Name\":\"ALBName\",\"Value\":\"$ALB_NAME\"}]},
+                {\"MetricName\":\"TargetGroupCount\",\"Value\":$TG_COUNT,\"Unit\":\"Count\",\"Dimensions\":[{\"Name\":\"ALBName\",\"Value\":\"$ALB_NAME\"}]}
+            ]" \
+            --region "$REGION" 2>&1
+
+        if [ $? -eq 0 ]; then
+            echo "✓ CloudWatch metrics published successfully"
+        else
+            echo "⚠ Failed to publish CloudWatch metrics (check IAM permissions for cloudwatch:PutMetricData)"
+        fi
+        ;;
+
+    datadog)
+        echo "Publishing to Datadog..."
+
+        if [ -z "$DATADOG_API_KEY" ]; then
+            echo "⚠ DATADOG_API_KEY not set, skipping Datadog metrics publishing"
+            return 0
+        fi
+
+        DATADOG_SITE="${DATADOG_SITE:-datadoghq.com}"
+        TIMESTAMP=$(date +%s)
+
+        RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -X POST "https://api.${DATADOG_SITE}/api/v2/series" \
+            -H "DD-API-KEY: $DATADOG_API_KEY" \
+            -H "Content-Type: application/json" \
+            -d "{
+                \"series\": [
+                    {
+                        \"metric\": \"nullplatform.alb.rule_count\",
+                        \"type\": 1,
+                        \"points\": [{\"timestamp\": $TIMESTAMP, \"value\": $TOTAL_RULES}],
+                        \"tags\": [\"alb_name:$ALB_NAME\", \"region:$REGION\"]
+                    },
+                    {
+                        \"metric\": \"nullplatform.alb.target_group_count\",
+                        \"type\": 1,
+                        \"points\": [{\"timestamp\": $TIMESTAMP, \"value\": $TG_COUNT}],
+                        \"tags\": [\"alb_name:$ALB_NAME\", \"region:$REGION\"]
+                    }
+                ]
+            }" 2>&1)
+
+        if [ "$RESPONSE" = "202" ]; then
+            echo "✓ Datadog metrics published successfully"
+        else
+            echo "⚠ Failed to publish Datadog metrics (HTTP $RESPONSE)"
+        fi
+        ;;
+
+    *)
+        echo "⚠ Unknown metrics target: $ALB_METRICS_PUBLISH_TARGET (expected: cloudwatch or datadog)"
+        ;;
+esac
+
+return 0
diff --git a/k8s/deployment/workflows/finalize.yaml b/k8s/deployment/workflows/finalize.yaml
index 178a396e..56b78305 100644
--- a/k8s/deployment/workflows/finalize.yaml
+++ b/k8s/deployment/workflows/finalize.yaml
@@ -45,11 +45,17 @@ steps:
         type: file
         file: "$OUTPUT_DIR/ingress-$SCOPE_ID-$DEPLOYMENT_ID.yaml"
     post:
-      name: verify_networking_reconciliation
-      type: script
-      file: "$SERVICE_PATH/deployment/verify_networking_reconciliation"
-      configuration:
-        VERIFY_WEIGHTS: false
+      name: post_apply_checks
+      type: workflow
+      steps:
+        - name: verify_networking_reconciliation
+          type: script
+          file: "$SERVICE_PATH/deployment/verify_networking_reconciliation"
+          configuration:
+            VERIFY_WEIGHTS: false
+        - name: publish_alb_metrics
+          type: script
+          file: "$SERVICE_PATH/deployment/publish_alb_metrics"
   - name: build deployment
     type: script
     file: "$SERVICE_PATH/deployment/build_blue_deployment"
diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml
index c00f0435..04533a94 100644
--- a/k8s/deployment/workflows/initial.yaml
+++ b/k8s/deployment/workflows/initial.yaml
@@ -67,4 +67,7 @@ steps:
           file: "$SERVICE_PATH/deployment/wait_deployment_active"
           configuration:
             TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS
+        - name: publish_alb_metrics
+          type: script
+          file: "$SERVICE_PATH/deployment/publish_alb_metrics"
 
diff --git a/k8s/deployment/workflows/switch_traffic.yaml b/k8s/deployment/workflows/switch_traffic.yaml
index 486cee7b..56be2790 100644
--- a/k8s/deployment/workflows/switch_traffic.yaml
+++ b/k8s/deployment/workflows/switch_traffic.yaml
@@ -48,8 +48,14 @@ steps:
       ACTION: apply
       DRY_RUN: false
     post:
-      name: verify_networking_reconciliation
-      type: script
-      file: "$SERVICE_PATH/deployment/verify_networking_reconciliation"
-      configuration:
-        VERIFY_WEIGHTS: true
+      name: post_apply_checks
+      type: workflow
+      steps:
+        - name: verify_networking_reconciliation
+          type: script
+          file: "$SERVICE_PATH/deployment/verify_networking_reconciliation"
+          configuration:
+            VERIFY_WEIGHTS: true
+        - name: publish_alb_metrics
+          type: script
+          file: "$SERVICE_PATH/deployment/publish_alb_metrics"
diff --git a/k8s/values.yaml b/k8s/values.yaml
index 56edaa68..f0c2df40 100644
--- a/k8s/values.yaml
+++ b/k8s/values.yaml
@@ -9,6 +9,8 @@ configuration:
   USE_ACCOUNT_SLUG: false
   DNS_TYPE: route53 # Available values route53 | azure | external_dns
   ALB_RECONCILIATION_ENABLED: false
+  ALB_METRICS_PUBLISH_ENABLED: false
+# ALB_METRICS_PUBLISH_TARGET: cloudwatch # Available values: cloudwatch | datadog
   DEPLOYMENT_MAX_WAIT_IN_SECONDS: 600
   DEPLOYMENT_TEMPLATE: "$SERVICE_PATH/deployment/templates/deployment.yaml.tpl"
   SECRET_TEMPLATE: "$SERVICE_PATH/deployment/templates/secret.yaml.tpl"

From e2fccc987f8f52ded920ab2994ae4e7964e12563 Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Thu, 9 Apr 2026 09:43:24 -0300
Subject: [PATCH 02/10] fix: align metric namespaces with AWS and Datadog
 conventions

- CloudWatch: nullplatform/ApplicationELB (mirrors AWS/ApplicationELB)
- Datadog: nullplatform.applicationelb.* (mirrors aws.applicationelb.*)
---
 k8s/deployment/publish_alb_metrics | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/k8s/deployment/publish_alb_metrics b/k8s/deployment/publish_alb_metrics
index 5bff54d2..1ecb91f3 100755
--- a/k8s/deployment/publish_alb_metrics
+++ b/k8s/deployment/publish_alb_metrics
@@ -77,10 +77,10 @@ echo "  Target Groups: $TG_COUNT"
 # Publish metrics based on target
 case "$ALB_METRICS_PUBLISH_TARGET" in
     cloudwatch)
-        echo "Publishing to CloudWatch (namespace: Nullplatform/ALB)..."
+        echo "Publishing to CloudWatch (namespace: nullplatform/ApplicationELB)..."
 
         aws cloudwatch put-metric-data \
-            --namespace "Nullplatform/ALB" \
+            --namespace "nullplatform/ApplicationELB" \
             --metric-data "[
                 {\"MetricName\":\"RuleCount\",\"Value\":$TOTAL_RULES,\"Unit\":\"Count\",\"Dimensions\":[{\"Name\":\"ALBName\",\"Value\":\"$ALB_NAME\"}]},
                 {\"MetricName\":\"TargetGroupCount\",\"Value\":$TG_COUNT,\"Unit\":\"Count\",\"Dimensions\":[{\"Name\":\"ALBName\",\"Value\":\"$ALB_NAME\"}]}
@@ -111,13 +111,13 @@ case "$ALB_METRICS_PUBLISH_TARGET" in
             -d "{
                 \"series\": [
                     {
-                        \"metric\": \"nullplatform.alb.rule_count\",
+                        \"metric\": \"nullplatform.applicationelb.rule_count\",
                         \"type\": 1,
                         \"points\": [{\"timestamp\": $TIMESTAMP, \"value\": $TOTAL_RULES}],
                         \"tags\": [\"alb_name:$ALB_NAME\", \"region:$REGION\"]
                     },
                     {
-                        \"metric\": \"nullplatform.alb.target_group_count\",
+                        \"metric\": \"nullplatform.applicationelb.target_group_count\",
                         \"type\": 1,
                         \"points\": [{\"timestamp\": $TIMESTAMP, \"value\": $TG_COUNT}],
                         \"tags\": [\"alb_name:$ALB_NAME\", \"region:$REGION\"]

From bf5eed114ce55a1653191f0ff4269f19a798b799 Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Thu, 9 Apr 2026 14:47:11 -0300
Subject: [PATCH 03/10] fix: decouple metrics publishing from
 ALB_RECONCILIATION_ENABLED

Metrics publishing now only depends on ALB_METRICS_PUBLISH_ENABLED
---
 k8s/deployment/publish_alb_metrics | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/k8s/deployment/publish_alb_metrics b/k8s/deployment/publish_alb_metrics
index 1ecb91f3..f5b9dde0 100755
--- a/k8s/deployment/publish_alb_metrics
+++ b/k8s/deployment/publish_alb_metrics
@@ -3,12 +3,11 @@
 # Publishes ALB rule count and target group count as custom metrics
 # to CloudWatch or Datadog for continuous monitoring and alerting.
 
-ALB_RECONCILIATION_ENABLED="${ALB_RECONCILIATION_ENABLED:-false}"
 ALB_METRICS_PUBLISH_ENABLED="${ALB_METRICS_PUBLISH_ENABLED:-false}"
 ALB_METRICS_PUBLISH_TARGET="${ALB_METRICS_PUBLISH_TARGET:-cloudwatch}"
 
-if [ "$ALB_RECONCILIATION_ENABLED" != "true" ] || [ "$ALB_METRICS_PUBLISH_ENABLED" != "true" ]; then
-    echo "ℹ ALB metrics publishing skipped (ALB_RECONCILIATION_ENABLED=$ALB_RECONCILIATION_ENABLED, ALB_METRICS_PUBLISH_ENABLED=$ALB_METRICS_PUBLISH_ENABLED)"
+if [ "$ALB_METRICS_PUBLISH_ENABLED" != "true" ]; then
+    echo "ℹ ALB metrics publishing skipped (ALB_METRICS_PUBLISH_ENABLED=$ALB_METRICS_PUBLISH_ENABLED)"
     return 0
 fi
 

From cfc9cb31fea840f70d6f337b3854183602e408b9 Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Thu, 9 Apr 2026 22:03:02 -0300
Subject: [PATCH 04/10] fix: publish ALB metrics before wait_deployment_active

Move publish_alb_metrics step right after verify_networking_reconciliation
so metrics reflect sooner instead of waiting for pods to be ready.
---
 k8s/deployment/workflows/initial.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/deployment/workflows/initial.yaml b/k8s/deployment/workflows/initial.yaml
index 04533a94..876c385d 100644
--- a/k8s/deployment/workflows/initial.yaml
+++ b/k8s/deployment/workflows/initial.yaml
@@ -62,12 +62,12 @@ steps:
           file: "$SERVICE_PATH/deployment/verify_networking_reconciliation"
           configuration:
             VERIFY_WEIGHTS: false
+        - name: publish_alb_metrics
+          type: script
+          file: "$SERVICE_PATH/deployment/publish_alb_metrics"
         - name: wait deployment active
           type: script
           file: "$SERVICE_PATH/deployment/wait_deployment_active"
           configuration:
             TIMEOUT: DEPLOYMENT_MAX_WAIT_IN_SECONDS
-        - name: publish_alb_metrics
-          type: script
-          file: "$SERVICE_PATH/deployment/publish_alb_metrics"
 

From 84608759977c8f44e1165891e8234b30872ca22c Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Thu, 9 Apr 2026 22:30:34 -0300
Subject: [PATCH 05/10] fix: reduce publish_alb_metrics log verbosity

Single line on success, single line on error, silent when disabled.
---
 k8s/deployment/publish_alb_metrics | 42 ++++++++++++------------------
 1 file changed, 16 insertions(+), 26 deletions(-)

diff --git a/k8s/deployment/publish_alb_metrics b/k8s/deployment/publish_alb_metrics
index f5b9dde0..5435e661 100755
--- a/k8s/deployment/publish_alb_metrics
+++ b/k8s/deployment/publish_alb_metrics
@@ -7,7 +7,6 @@ ALB_METRICS_PUBLISH_ENABLED="${ALB_METRICS_PUBLISH_ENABLED:-false}"
 ALB_METRICS_PUBLISH_TARGET="${ALB_METRICS_PUBLISH_TARGET:-cloudwatch}"
 
 if [ "$ALB_METRICS_PUBLISH_ENABLED" != "true" ]; then
-    echo "ℹ ALB metrics publishing skipped (ALB_METRICS_PUBLISH_ENABLED=$ALB_METRICS_PUBLISH_ENABLED)"
     return 0
 fi
 
@@ -15,21 +14,19 @@ ALB_NAME=$(echo "$CONTEXT" | jq -r '.alb_name')
 REGION=$(echo "$CONTEXT" | jq -r '.region')
 
 if [ -z "$ALB_NAME" ] || [ "$ALB_NAME" = "null" ]; then
-    echo "⚠ ALB name not found in context, skipping metrics publishing"
+    echo "⚠ ALB metrics: ALB name not found in context"
     return 0
 fi
 
-echo "Publishing ALB metrics for [$ALB_NAME] in region [$REGION]..."
-
 # Resolve ALB ARN
 ALB_ARN=$(aws elbv2 describe-load-balancers \
     --names "$ALB_NAME" \
     --region "$REGION" \
     --query 'LoadBalancers[0].LoadBalancerArn' \
-    --output text 2>&1)
+    --output text 2>/dev/null)
 
 if [ $? -ne 0 ] || [ "$ALB_ARN" = "None" ] || [ -z "$ALB_ARN" ]; then
-    echo "⚠ Could not find ALB [$ALB_NAME], skipping metrics publishing"
+    echo "⚠ ALB metrics: could not find ALB [$ALB_NAME]"
     return 0
 fi
 
@@ -38,10 +35,10 @@ TOTAL_RULES=0
 LISTENERS=$(aws elbv2 describe-listeners \
     --load-balancer-arn "$ALB_ARN" \
     --region "$REGION" \
-    --output json 2>&1)
+    --output json 2>/dev/null)
 
 if [ $? -ne 0 ]; then
-    echo "⚠ Could not retrieve listeners for ALB [$ALB_NAME], skipping metrics publishing"
+    echo "⚠ ALB metrics: could not retrieve listeners"
     return 0
 fi
 
@@ -51,7 +48,7 @@ for listener_arn in $LISTENER_ARNS; do
     RULES=$(aws elbv2 describe-rules \
         --listener-arn "$listener_arn" \
         --region "$REGION" \
-        --output json 2>&1)
+        --output json 2>/dev/null)
 
     if [ $? -eq 0 ]; then
         LISTENER_RULE_COUNT=$(echo "$RULES" | jq '[.Rules[] | select(.IsDefault != true)] | length')
@@ -63,41 +60,34 @@ done
 TARGET_GROUPS=$(aws elbv2 describe-target-groups \
     --load-balancer-arn "$ALB_ARN" \
     --region "$REGION" \
-    --output json 2>&1)
+    --output json 2>/dev/null)
 
 TG_COUNT=0
 if [ $? -eq 0 ]; then
     TG_COUNT=$(echo "$TARGET_GROUPS" | jq '.TargetGroups | length')
 fi
 
-echo "  Rules: $TOTAL_RULES"
-echo "  Target Groups: $TG_COUNT"
-
-# Publish metrics based on target
+# Publish metrics
 case "$ALB_METRICS_PUBLISH_TARGET" in
     cloudwatch)
-        echo "Publishing to CloudWatch (namespace: nullplatform/ApplicationELB)..."
-
         aws cloudwatch put-metric-data \
             --namespace "nullplatform/ApplicationELB" \
             --metric-data "[
                 {\"MetricName\":\"RuleCount\",\"Value\":$TOTAL_RULES,\"Unit\":\"Count\",\"Dimensions\":[{\"Name\":\"ALBName\",\"Value\":\"$ALB_NAME\"}]},
                 {\"MetricName\":\"TargetGroupCount\",\"Value\":$TG_COUNT,\"Unit\":\"Count\",\"Dimensions\":[{\"Name\":\"ALBName\",\"Value\":\"$ALB_NAME\"}]}
             ]" \
-            --region "$REGION" 2>&1
+            --region "$REGION" 2>/dev/null
 
         if [ $? -eq 0 ]; then
-            echo "✓ CloudWatch metrics published successfully"
+            echo "✓ ALB metrics published to CloudWatch (rules: $TOTAL_RULES, target_groups: $TG_COUNT)"
         else
-            echo "⚠ Failed to publish CloudWatch metrics (check IAM permissions for cloudwatch:PutMetricData)"
+            echo "⚠ ALB metrics: failed to publish to CloudWatch"
         fi
         ;;
 
     datadog)
-        echo "Publishing to Datadog..."
-
         if [ -z "$DATADOG_API_KEY" ]; then
-            echo "⚠ DATADOG_API_KEY not set, skipping Datadog metrics publishing"
+            echo "⚠ ALB metrics: DATADOG_API_KEY not set"
             return 0
         fi
 
@@ -122,17 +112,17 @@ case "$ALB_METRICS_PUBLISH_TARGET" in
                         \"tags\": [\"alb_name:$ALB_NAME\", \"region:$REGION\"]
                     }
                 ]
-            }" 2>&1)
+            }" 2>/dev/null)
 
         if [ "$RESPONSE" = "202" ]; then
-            echo "✓ Datadog metrics published successfully"
+            echo "✓ ALB metrics published to Datadog (rules: $TOTAL_RULES, target_groups: $TG_COUNT)"
         else
-            echo "⚠ Failed to publish Datadog metrics (HTTP $RESPONSE)"
+            echo "⚠ ALB metrics: failed to publish to Datadog (HTTP $RESPONSE)"
         fi
         ;;
 
     *)
-        echo "⚠ Unknown metrics target: $ALB_METRICS_PUBLISH_TARGET (expected: cloudwatch or datadog)"
+        echo "⚠ ALB metrics: unknown target '$ALB_METRICS_PUBLISH_TARGET'"
         ;;
 esac
 

From 7f37753dd2ab0723399b837ac056bdf7fba7149a Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Fri, 10 Apr 2026 09:28:59 -0300
Subject: [PATCH 06/10] feat: add testing framework and BATS tests for
 publish_alb_metrics

- Add scope-testing submodule
- Add 17 unit tests covering CloudWatch, Datadog, error handling,
  skip logic, and rule counting across multiple listeners
---
 .gitmodules                                   |   3 +
 .../tests/scripts/publish_alb_metrics.bats    | 267 ++++++++++++++++++
 testing                                       |   1 +
 3 files changed, 271 insertions(+)
 create mode 100644 .gitmodules
 create mode 100644 k8s/deployment/tests/scripts/publish_alb_metrics.bats
 create mode 160000 testing

diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 00000000..9b360f07
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "testing"]
+	path = testing
+	url = git@github.com:nullplatform/scope-testing.git
diff --git a/k8s/deployment/tests/scripts/publish_alb_metrics.bats b/k8s/deployment/tests/scripts/publish_alb_metrics.bats
new file mode 100644
index 00000000..29bc2d45
--- /dev/null
+++ b/k8s/deployment/tests/scripts/publish_alb_metrics.bats
@@ -0,0 +1,267 @@
+#!/usr/bin/env bats
+
+setup() {
+  PROJECT_ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../../.." && pwd)"
+  source "$PROJECT_ROOT/testing/assertions.sh"
+
+  export SCRIPT="$PROJECT_ROOT/k8s/deployment/publish_alb_metrics"
+
+  # Default context
+  export CONTEXT='{"alb_name":"k8s-nullplatform-internet-facing","region":"us-east-1"}'
+
+  # Default config
+  export ALB_METRICS_PUBLISH_ENABLED="true"
+  export ALB_METRICS_PUBLISH_TARGET="cloudwatch"
+
+  # Track calls
+  export AWS_CALLS_LOG="$BATS_TEST_TMPDIR/aws_calls.log"
+  export CURL_CALLS_LOG="$BATS_TEST_TMPDIR/curl_calls.log"
+
+  # Mock aws CLI
+  aws() {
+    echo "$*" >> "$AWS_CALLS_LOG"
+    case "$*" in
+      *"describe-load-balancers"*)
+        echo "arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/k8s-nullplatform-internet-facing/abc123"
+        ;;
+      *"describe-listeners"*)
+        echo '{"Listeners":[{"ListenerArn":"arn:aws:elasticloadbalancing:us-east-1:123456789:listener/app/abc/123"}]}'
+        ;;
+      *"describe-rules"*)
+        echo '{"Rules":[{"IsDefault":true},{"IsDefault":false},{"IsDefault":false},{"IsDefault":false}]}'
+        ;;
+      *"describe-target-groups"*)
+        echo '{"TargetGroups":[{},{},{},{},{}]}'
+        ;;
+      *"put-metric-data"*)
+        return 0
+        ;;
+    esac
+  }
+  export -f aws
+
+  # Mock curl
+  curl() {
+    echo "$*" >> "$CURL_CALLS_LOG"
+    echo "202"
+  }
+  export -f curl
+}
+
+run_script() {
+  run bash -c 'source "$SCRIPT"'
+}
+
+# =============================================================================
+# Disabled / skipped scenarios
+# =============================================================================
+
+@test "skips silently when ALB_METRICS_PUBLISH_ENABLED is false" {
+  export ALB_METRICS_PUBLISH_ENABLED="false"
+  run_script
+  assert_equal "$status" "0"
+  assert_equal "$output" ""
+}
+
+@test "skips silently when ALB_METRICS_PUBLISH_ENABLED is not set" {
+  unset ALB_METRICS_PUBLISH_ENABLED
+  run_script
+  assert_equal "$status" "0"
+  assert_equal "$output" ""
+}
+
+# =============================================================================
+# Error scenarios
+# =============================================================================
+
+@test "warns when ALB name not found in context" {
+  export CONTEXT='{"region":"us-east-1"}'
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: ALB name not found in context"
+}
+
+@test "warns when ALB name is null in context" {
+  export CONTEXT='{"alb_name":null,"region":"us-east-1"}'
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: ALB name not found in context"
+}
+
+@test "warns when ALB not found in AWS" {
+  aws() {
+    case "$*" in
+      *"describe-load-balancers"*) echo "None" ;;
+    esac
+  }
+  export -f aws
+
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: could not find ALB"
+}
+
+@test "warns when describe-load-balancers fails" {
+  aws() {
+    case "$*" in
+      *"describe-load-balancers"*) return 1 ;;
+    esac
+  }
+  export -f aws
+
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: could not find ALB"
+}
+
+@test "warns when describe-listeners fails" {
+  aws() {
+    case "$*" in
+      *"describe-load-balancers"*) echo "arn:aws:elasticloadbalancing:us-east-1:123:lb/abc" ;;
+      *"describe-listeners"*) return 1 ;;
+    esac
+  }
+  export -f aws
+
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: could not retrieve listeners"
+}
+
+# =============================================================================
+# CloudWatch success
+# =============================================================================
+
+@test "publishes to CloudWatch with correct rule and target group counts" {
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics published to CloudWatch (rules: 3, target_groups: 5)"
+}
+
+@test "CloudWatch put-metric-data uses correct namespace and dimensions" {
+  run_script
+  local calls=$(cat "$AWS_CALLS_LOG")
+  assert_contains "$calls" "nullplatform/ApplicationELB"
+  assert_contains "$calls" "k8s-nullplatform-internet-facing"
+  assert_contains "$calls" "RuleCount"
+  assert_contains "$calls" "TargetGroupCount"
+}
+
+@test "warns when CloudWatch put-metric-data fails" {
+  aws() {
+    echo "$*" >> "$AWS_CALLS_LOG"
+    case "$*" in
+      *"describe-load-balancers"*) echo "arn:aws:elasticloadbalancing:us-east-1:123:lb/abc" ;;
+      *"describe-listeners"*) echo '{"Listeners":[{"ListenerArn":"arn:listener/123"}]}' ;;
+      *"describe-rules"*) echo '{"Rules":[{"IsDefault":true}]}' ;;
+      *"describe-target-groups"*) echo '{"TargetGroups":[]}' ;;
+      *"put-metric-data"*) return 1 ;;
+    esac
+  }
+  export -f aws
+
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: failed to publish to CloudWatch"
+}
+
+# =============================================================================
+# Datadog success
+# =============================================================================
+
+@test "publishes to Datadog with correct counts" {
+  export ALB_METRICS_PUBLISH_TARGET="datadog"
+  export DATADOG_API_KEY="test-api-key"
+  export DATADOG_SITE="datadoghq.com"
+
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics published to Datadog (rules: 3, target_groups: 5)"
+}
+
+@test "Datadog request uses correct endpoint and metric names" {
+  export ALB_METRICS_PUBLISH_TARGET="datadog"
+  export DATADOG_API_KEY="test-api-key"
+  export DATADOG_SITE="datadoghq.eu"
+
+  run_script
+  local calls=$(cat "$CURL_CALLS_LOG")
+  assert_contains "$calls" "https://api.datadoghq.eu/api/v2/series"
+  assert_contains "$calls" "nullplatform.applicationelb.rule_count"
+  assert_contains "$calls" "nullplatform.applicationelb.target_group_count"
+  assert_contains "$calls" "alb_name:k8s-nullplatform-internet-facing"
+}
+
+@test "warns when DATADOG_API_KEY not set" {
+  export ALB_METRICS_PUBLISH_TARGET="datadog"
+  unset DATADOG_API_KEY
+
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: DATADOG_API_KEY not set"
+}
+
+@test "warns when Datadog returns non-202" {
+  export ALB_METRICS_PUBLISH_TARGET="datadog"
+  export DATADOG_API_KEY="test-api-key"
+
+  curl() {
+    echo "403"
+  }
+  export -f curl
+
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: failed to publish to Datadog (HTTP 403)"
+}
+
+# =============================================================================
+# Unknown target
+# =============================================================================
+
+@test "warns on unknown metrics target" {
+  export ALB_METRICS_PUBLISH_TARGET="prometheus"
+  run_script
+  assert_equal "$status" "0"
+  assert_contains "$output" "ALB metrics: unknown target 'prometheus'"
+}
+
+# =============================================================================
+# Rule counting logic
+# =============================================================================
+
+@test "excludes default rules from count" {
+  aws() {
+    echo "$*" >> "$AWS_CALLS_LOG"
+    case "$*" in
+      *"describe-load-balancers"*) echo "arn:aws:elasticloadbalancing:us-east-1:123:lb/abc" ;;
+      *"describe-listeners"*) echo '{"Listeners":[{"ListenerArn":"arn:listener/123"}]}' ;;
+      *"describe-rules"*) echo '{"Rules":[{"IsDefault":true},{"IsDefault":false}]}' ;;
+      *"describe-target-groups"*) echo '{"TargetGroups":[{}]}' ;;
+      *"put-metric-data"*) return 0 ;;
+    esac
+  }
+  export -f aws
+
+  run_script
+  assert_contains "$output" "rules: 1, target_groups: 1"
+}
+
+@test "counts rules across multiple listeners" {
+  aws() {
+    echo "$*" >> "$AWS_CALLS_LOG"
+    case "$*" in
+      *"describe-load-balancers"*) echo "arn:aws:elasticloadbalancing:us-east-1:123:lb/abc" ;;
+      *"describe-listeners"*) echo '{"Listeners":[{"ListenerArn":"arn:listener/1"},{"ListenerArn":"arn:listener/2"}]}' ;;
+      *"describe-rules"*"listener/1"*) echo '{"Rules":[{"IsDefault":true},{"IsDefault":false},{"IsDefault":false}]}' ;;
+      *"describe-rules"*"listener/2"*) echo '{"Rules":[{"IsDefault":true},{"IsDefault":false}]}' ;;
+      *"describe-rules"*) echo '{"Rules":[{"IsDefault":true},{"IsDefault":false},{"IsDefault":false}]}' ;;
+      *"describe-target-groups"*) echo '{"TargetGroups":[{},{}]}' ;;
+      *"put-metric-data"*) return 0 ;;
+    esac
+  }
+  export -f aws
+
+  run_script
+  assert_contains "$output" "rules: 3, target_groups: 2"
+}
diff --git a/testing b/testing
new file mode 160000
index 00000000..30391d90
--- /dev/null
+++ b/testing
@@ -0,0 +1 @@
+Subproject commit 30391d9050b0e0a3e377bec9cd5a6283e67acb20

From 18ad6dd10d660ef5008b6833d672c2fc62c1c011 Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Fri, 10 Apr 2026 10:16:20 -0300
Subject: [PATCH 07/10] fix: move test to match beta convention (tests/ not
 tests/scripts/)

---
 k8s/deployment/tests/{scripts => }/publish_alb_metrics.bats | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename k8s/deployment/tests/{scripts => }/publish_alb_metrics.bats (99%)

diff --git a/k8s/deployment/tests/scripts/publish_alb_metrics.bats b/k8s/deployment/tests/publish_alb_metrics.bats
similarity index 99%
rename from k8s/deployment/tests/scripts/publish_alb_metrics.bats
rename to k8s/deployment/tests/publish_alb_metrics.bats
index 29bc2d45..04a7ff30 100644
--- a/k8s/deployment/tests/scripts/publish_alb_metrics.bats
+++ b/k8s/deployment/tests/publish_alb_metrics.bats
@@ -1,7 +1,7 @@
 #!/usr/bin/env bats
 
 setup() {
-  PROJECT_ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../../.." && pwd)"
+  PROJECT_ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)"
   source "$PROJECT_ROOT/testing/assertions.sh"
 
   export SCRIPT="$PROJECT_ROOT/k8s/deployment/publish_alb_metrics"

From 3dffb2db3f400d90ccc60b324bebac8c2e4fd797 Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Fri, 10 Apr 2026 10:22:25 -0300
Subject: [PATCH 08/10] fix: remove .gitmodules (testing already exists in beta
 as files)

---
 .gitmodules | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 .gitmodules

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 9b360f07..00000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "testing"]
-	path = testing
-	url = git@github.com:nullplatform/scope-testing.git

From 1842517d1db96c2cfdac2ffa58c218db4b078485 Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Fri, 10 Apr 2026 11:05:29 -0300
Subject: [PATCH 09/10] fix: adopt log convention with level filtering and
 emojis

---
 k8s/deployment/publish_alb_metrics            | 18 +++++++++---------
 k8s/deployment/tests/publish_alb_metrics.bats |  8 ++++++++
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/k8s/deployment/publish_alb_metrics b/k8s/deployment/publish_alb_metrics
index 5435e661..1b7bb69c 100755
--- a/k8s/deployment/publish_alb_metrics
+++ b/k8s/deployment/publish_alb_metrics
@@ -14,7 +14,7 @@ ALB_NAME=$(echo "$CONTEXT" | jq -r '.alb_name')
 REGION=$(echo "$CONTEXT" | jq -r '.region')
 
 if [ -z "$ALB_NAME" ] || [ "$ALB_NAME" = "null" ]; then
-    echo "⚠ ALB metrics: ALB name not found in context"
+    log warn "⚠️ ALB metrics: ALB name not found in context"
     return 0
 fi
 
@@ -26,7 +26,7 @@ ALB_ARN=$(aws elbv2 describe-load-balancers \
     --output text 2>/dev/null)
 
 if [ $? -ne 0 ] || [ "$ALB_ARN" = "None" ] || [ -z "$ALB_ARN" ]; then
-    echo "⚠ ALB metrics: could not find ALB [$ALB_NAME]"
+    log warn "⚠️ ALB metrics: could not find ALB [$ALB_NAME]"
     return 0
 fi
 
@@ -38,7 +38,7 @@ LISTENERS=$(aws elbv2 describe-listeners \
     --output json 2>/dev/null)
 
 if [ $? -ne 0 ]; then
-    echo "⚠ ALB metrics: could not retrieve listeners"
+    log warn "⚠️ ALB metrics: could not retrieve listeners"
     return 0
 fi
 
@@ -79,15 +79,15 @@ case "$ALB_METRICS_PUBLISH_TARGET" in
             --region "$REGION" 2>/dev/null
 
         if [ $? -eq 0 ]; then
-            echo "✓ ALB metrics published to CloudWatch (rules: $TOTAL_RULES, target_groups: $TG_COUNT)"
+            log info "✅ ALB metrics published to CloudWatch (rules: $TOTAL_RULES, target_groups: $TG_COUNT)"
         else
-            echo "⚠ ALB metrics: failed to publish to CloudWatch"
+            log error "❌ ALB metrics: failed to publish to CloudWatch"
         fi
         ;;
 
     datadog)
         if [ -z "$DATADOG_API_KEY" ]; then
-            echo "⚠ ALB metrics: DATADOG_API_KEY not set"
+            log warn "⚠️ ALB metrics: DATADOG_API_KEY not set"
             return 0
         fi
 
@@ -115,14 +115,14 @@ case "$ALB_METRICS_PUBLISH_TARGET" in
             }" 2>/dev/null)
 
         if [ "$RESPONSE" = "202" ]; then
-            echo "✓ ALB metrics published to Datadog (rules: $TOTAL_RULES, target_groups: $TG_COUNT)"
+            log info "✅ ALB metrics published to Datadog (rules: $TOTAL_RULES, target_groups: $TG_COUNT)"
         else
-            echo "⚠ ALB metrics: failed to publish to Datadog (HTTP $RESPONSE)"
+            log error "❌ ALB metrics: failed to publish to Datadog (HTTP $RESPONSE)"
         fi
         ;;
 
     *)
-        echo "⚠ ALB metrics: unknown target '$ALB_METRICS_PUBLISH_TARGET'"
+        log warn "⚠️ ALB metrics: unknown target '$ALB_METRICS_PUBLISH_TARGET'"
         ;;
 esac
 
diff --git a/k8s/deployment/tests/publish_alb_metrics.bats b/k8s/deployment/tests/publish_alb_metrics.bats
index 04a7ff30..32f777e9 100644
--- a/k8s/deployment/tests/publish_alb_metrics.bats
+++ b/k8s/deployment/tests/publish_alb_metrics.bats
@@ -46,6 +46,14 @@ setup() {
     echo "202"
   }
   export -f curl
+
+  # Mock log function (from k8s/logging)
+  log() {
+    local level="${1:-info}"
+    local message="${2:-}"
+    echo "$message"
+  }
+  export -f log
 }
 
 run_script() {

From 1a3f52a15221a7f847fde71dd425a81e86a4b000 Mon Sep 17 00:00:00 2001
From: Pablo Ilundain <pablo.ilundain@nullplatform.io>
Date: Fri, 10 Apr 2026 11:23:19 -0300
Subject: [PATCH 10/10] feat: use get_config_value for ALB metrics
 configuration

Support reading ALB_METRICS_PUBLISH_ENABLED and ALB_METRICS_PUBLISH_TARGET
from scope-configurations provider, env vars, or defaults.
---
 k8s/deployment/publish_alb_metrics            | 16 ++++++++++++++--
 k8s/deployment/tests/publish_alb_metrics.bats |  4 ++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/k8s/deployment/publish_alb_metrics b/k8s/deployment/publish_alb_metrics
index 1b7bb69c..fd695b77 100755
--- a/k8s/deployment/publish_alb_metrics
+++ b/k8s/deployment/publish_alb_metrics
@@ -3,8 +3,20 @@
 # Publishes ALB rule count and target group count as custom metrics
 # to CloudWatch or Datadog for continuous monitoring and alerting.
 
-ALB_METRICS_PUBLISH_ENABLED="${ALB_METRICS_PUBLISH_ENABLED:-false}"
-ALB_METRICS_PUBLISH_TARGET="${ALB_METRICS_PUBLISH_TARGET:-cloudwatch}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../utils/get_config_value"
+
+ALB_METRICS_PUBLISH_ENABLED=$(get_config_value \
+  --env ALB_METRICS_PUBLISH_ENABLED \
+  --provider '.providers["scope-configurations"].networking.alb_metrics_enabled' \
+  --default "false"
+)
+
+ALB_METRICS_PUBLISH_TARGET=$(get_config_value \
+  --env ALB_METRICS_PUBLISH_TARGET \
+  --provider '.providers["scope-configurations"].networking.alb_metrics_target' \
+  --default "cloudwatch"
+)
 
 if [ "$ALB_METRICS_PUBLISH_ENABLED" != "true" ]; then
     return 0
diff --git a/k8s/deployment/tests/publish_alb_metrics.bats b/k8s/deployment/tests/publish_alb_metrics.bats
index 32f777e9..48f85d03 100644
--- a/k8s/deployment/tests/publish_alb_metrics.bats
+++ b/k8s/deployment/tests/publish_alb_metrics.bats
@@ -47,6 +47,10 @@ setup() {
   }
   export -f curl
 
+  # Source real get_config_value (uses CONTEXT + env vars already set)
+  source "$PROJECT_ROOT/k8s/utils/get_config_value"
+  export -f get_config_value
+
   # Mock log function (from k8s/logging)
   log() {
     local level="${1:-info}"