From 787981970e7e4e75669d816ef38c8d1359a1a7f7 Mon Sep 17 00:00:00 2001
From: Peter Burkholder <peter.burkholder@gsa.gov>
Date: Wed, 31 Dec 2025 12:25:19 -0500
Subject: [PATCH 1/4] Progressing w/ PGAudit enablement

---
 .gitignore                |  2 ++
 aws/rds/enable-pgaudit.sh | 61 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100755 aws/rds/enable-pgaudit.sh

diff --git a/.gitignore b/.gitignore
index ce3f5e85..c4ac60ba 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,7 @@ my*.yml
 *.xlsx
 .vscode
 venv
+.venv
 **/__pycache__
 *.txt
 
@@ -30,3 +31,4 @@ certbot/work/**
 !certbot/work/.gitkeep
 
 concourse/*.yml
+__pycache__
\ No newline at end of file
diff --git a/aws/rds/enable-pgaudit.sh b/aws/rds/enable-pgaudit.sh
new file mode 100755
index 00000000..b6a9b09a
--- /dev/null
+++ b/aws/rds/enable-pgaudit.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+#org=epa-spds
+#space=test
+#service_instance=spds-db
+
+org=cloud-gov-operators
+space=peter.burkholder
+service_instance=mia-django-db-x-psql15
+
+echo "Service Instance: $service_instance in Org: $org, Space: $space"
+
+cf target -o $org -s $space > /dev/null
+service_instance_guid=$(cf service $service_instance --guid)
+echo "Service Instance GUID: $service_instance_guid"
+
+# get ARN
+arn=$(aws resourcegroupstaggingapi get-resources \
+  --resource-type-filters "rds:db" \
+  --tag-filters Key="Instance GUID",Values="$service_instance_guid" \
+  | jq -r '.ResourceTagMappingList[].ResourceARN' | grep -v replica  )
+
+# get AWS instancea name from ARN
+instance=$(echo $arn | awk -F: '{print $NF}')
+echo "RDS Instance: $instance"
+
+# determine current parameter group
+current_parameter_group=$(aws rds describe-db-instances \
+  --db-instance-identifier $instance \
+  --query 'DBInstances[0].DBParameterGroups[?DBParameterGroupName!=`default`].DBParameterGroupName' \
+  --output text)
+
+echo "Current parameter group: $current_parameter_group"
+
+# Determine whether we need to create a new parameter group or modify existing one
+case $current_parameter_group in
+  "" )
+    echo "No custom parameter group associated with instance. Exiting."
+    exit 0
+    ;;
+  *default* )
+    echo "Using default parameter group $current_parameter_group. Exiting."
+    exit 0
+    ;;
+  *$service_instance* )
+    echo "Parameter group already set to $service_instance_guid. Exiting."
+    exit 0
+    ;;
+  * )
+    echo "Unknown parameter group $current_parameter_group. Exiting."
+    exit 0
+    ;;
+esac    
+
+
+#    aws rds modify-db-instance \
+#      --db-instance-identifier $instance \
+#      --db-parameter-group-name $service_instance_guid \
+#      --apply-immediately

From af6d25a35d6599d2114d62c2f1d02718d21db607 Mon Sep 17 00:00:00 2001
From: Peter Burkholder <peter.burkholder@gsa.gov>
Date: Wed, 31 Dec 2025 12:55:06 -0500
Subject: [PATCH 2/4] Script seems to do the right thing in echos

---
 aws/rds/enable-pgaudit.sh | 95 ++++++++++++++++++++++++++++++++-------
 1 file changed, 80 insertions(+), 15 deletions(-)

diff --git a/aws/rds/enable-pgaudit.sh b/aws/rds/enable-pgaudit.sh
index b6a9b09a..2ded1155 100755
--- a/aws/rds/enable-pgaudit.sh
+++ b/aws/rds/enable-pgaudit.sh
@@ -2,6 +2,64 @@
 
 set -eu -o pipefail
 
+function reboot_instance() {
+  [ $# -ne 1 ] && echo "Function: reboot_instance <db_instance>" && exit 1
+  local db_instance=$1
+
+  echo aws rds reboot-db-instance \
+    --db-instance-identifier "$db_instance"
+}
+
+function create_and_associate_parameter_group() {
+  [ $# -ne 2 ] && echo "Function: create_parameter_group <current_parameter_group> <db_instance>" && exit 1
+  local current_parameter_group=$1
+  local db_instance=$2
+
+  family=$(echo "${current_parameter_group}" | awk -F. '{print $NF}')
+
+  # create new parameter group
+  echo aws rds create-db-parameter-group \
+    --db-parameter-group-name "$db_instance" \
+    --db-parameter-group-family "$family" \
+    --description "Parameter group with pgaudit enabled"
+
+  setup_pgaudit_parameter "$db_instance"
+
+  # associate new parameter group with instance
+  echo aws rds modify-db-instance \
+    --db-instance-identifier "$db_instance" \
+    --db-parameter-group-name "$db_instance" \
+    --apply-immediately
+}
+
+
+function setup_pgaudit_parameter() {
+  [ $# -ne 1 ] && echo "Function: setup_pgaudit <parameter_group>" && exit 1
+  local parameter_group=$1
+
+  # modify pgaudit settings
+  echo aws rds modify-db-parameter-group \
+    --db-parameter-group-name "$parameter_group" \
+    --parameters "ParameterName=shared_preload_libraries,ParameterValue=pgaudit,ApplyMethod=pending-reboot" 
+}
+
+# check if parameter group already has pgaudit enabled
+function pgaudit_is_enabled() {
+  [ $# -ne 1 ] && echo "Function: check_pgaudit_enabled <parameter_group>" && exit 1
+  local parameter_group=$1
+
+  pgaudit_value=$(aws rds describe-db-parameters \
+    --db-parameter-group-name "$parameter_group" \
+    --query "Parameters[?ParameterName=='shared_preload_libraries'].ParameterValue" \
+    --output text)
+
+  if [[ "$pgaudit_value" == *pgaudit* ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
 #org=epa-spds
 #space=test
 #service_instance=spds-db
@@ -23,12 +81,12 @@ arn=$(aws resourcegroupstaggingapi get-resources \
   | jq -r '.ResourceTagMappingList[].ResourceARN' | grep -v replica  )
 
 # get AWS instancea name from ARN
-instance=$(echo $arn | awk -F: '{print $NF}')
-echo "RDS Instance: $instance"
+db_instance=$(echo $arn | awk -F: '{print $NF}')
+echo "RDS Instance: $db_instance"
 
 # determine current parameter group
 current_parameter_group=$(aws rds describe-db-instances \
-  --db-instance-identifier $instance \
+  --db-instance-identifier "$db_instance" \
   --query 'DBInstances[0].DBParameterGroups[?DBParameterGroupName!=`default`].DBParameterGroupName' \
   --output text)
 
@@ -37,25 +95,32 @@ echo "Current parameter group: $current_parameter_group"
 # Determine whether we need to create a new parameter group or modify existing one
 case $current_parameter_group in
   "" )
-    echo "No custom parameter group associated with instance. Exiting."
-    exit 0
+    echo "No custom parameter group associated with instance. Failing..."
+    exit 1
     ;;
   *default* )
-    echo "Using default parameter group $current_parameter_group. Exiting."
-    exit 0
+    echo "Uses default parameter group $current_parameter_group."
+    echo "Creating new parameter group $db_instance and associating it with instance."
+    create_and_associate_parameter_group "$current_parameter_group" "$db_instance"
+    reboot_instance "$db_instance"
     ;;
   *$service_instance* )
-    echo "Parameter group already set to $service_instance_guid. Exiting."
+    echo "Parameter group already set to $service_instance_guid."
+    if pgaudit_is_enabled "$current_parameter_group"; then
+      echo "pgaudit is already enabled in parameter group $current_parameter_group. Exiting."
+      exit 0
+    else
+      echo "Enabling pgaudit in parameter group $current_parameter_group."
+      setup_pgaudit_parameter "$current_parameter_group"
+      reboot_instance "$db_instance"
+    fi
     exit 0
     ;;
   * )
-    echo "Unknown parameter group $current_parameter_group. Exiting."
-    exit 0
+    echo "Unknown parameter group $current_parameter_group. Failing..."
+    exit 1
     ;;
 esac    
 
-
-#    aws rds modify-db-instance \
-#      --db-instance-identifier $instance \
-#      --db-parameter-group-name $service_instance_guid \
-#      --apply-immediately
+echo "Well, how did we get here?"
+exit 0
\ No newline at end of file

From 3658b0c0921153def5448ca448040c1e8b6ae079 Mon Sep 17 00:00:00 2001
From: Peter Burkholder <peter.burkholder@gsa.gov>
Date: Wed, 31 Dec 2025 13:21:57 -0500
Subject: [PATCH 3/4] Tested in peter.burkhodler

---
 aws/rds/enable-pgaudit.sh | 48 +++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 12 deletions(-)

diff --git a/aws/rds/enable-pgaudit.sh b/aws/rds/enable-pgaudit.sh
index 2ded1155..f1b52f41 100755
--- a/aws/rds/enable-pgaudit.sh
+++ b/aws/rds/enable-pgaudit.sh
@@ -3,11 +3,33 @@
 set -eu -o pipefail
 
 function reboot_instance() {
-  [ $# -ne 1 ] && echo "Function: reboot_instance <db_instance>" && exit 1
+  [ $# -ne 2 ] && echo "Function: reboot_instance <db_instance> <attempt>" && exit 1
   local db_instance=$1
+  local attempt=$2
 
-  echo aws rds reboot-db-instance \
+  aws rds reboot-db-instance \
     --db-instance-identifier "$db_instance"
+  
+  status=$?
+
+  if [ $status -eq 254 ]; then
+    echo "Failed to reboot instance $db_instance status code 254"
+    if [ -z "${attempt+x}" ]; then
+      attempt=1
+    else
+      attempt=$((attempt + 1))
+    fi
+    if [ $attempt -gt 5 ]; then
+      echo "Exceeded maximum reboot attempts. Exiting."
+      exit 1
+    fi
+    echo "Sleeping 30 seconds and trying again..."
+    sleep 30
+    reboot_instance "$db_instance" $attempt
+  elif [ $status -ne 0 ]; then
+    echo "Failed to reboot instance $db_instance with status code $status. Exiting." 
+    exit 1
+  fi
 }
 
 function create_and_associate_parameter_group() {
@@ -17,16 +39,16 @@ function create_and_associate_parameter_group() {
 
   family=$(echo "${current_parameter_group}" | awk -F. '{print $NF}')
 
-  # create new parameter group
-  echo aws rds create-db-parameter-group \
+  echo ============ create new parameter group ===========
+  aws rds create-db-parameter-group \
     --db-parameter-group-name "$db_instance" \
     --db-parameter-group-family "$family" \
     --description "Parameter group with pgaudit enabled"
 
   setup_pgaudit_parameter "$db_instance"
 
-  # associate new parameter group with instance
-  echo aws rds modify-db-instance \
+  echo =========== associate new parameter group with instance ===========
+  aws rds modify-db-instance \
     --db-instance-identifier "$db_instance" \
     --db-parameter-group-name "$db_instance" \
     --apply-immediately
@@ -37,8 +59,8 @@ function setup_pgaudit_parameter() {
   [ $# -ne 1 ] && echo "Function: setup_pgaudit <parameter_group>" && exit 1
   local parameter_group=$1
 
-  # modify pgaudit settings
-  echo aws rds modify-db-parameter-group \
+  echo =========== modify pgaudit settings in parameter group ===========
+  aws rds modify-db-parameter-group \
     --db-parameter-group-name "$parameter_group" \
     --parameters "ParameterName=shared_preload_libraries,ParameterValue=pgaudit,ApplyMethod=pending-reboot" 
 }
@@ -54,8 +76,10 @@ function pgaudit_is_enabled() {
     --output text)
 
   if [[ "$pgaudit_value" == *pgaudit* ]]; then
+    echo "pgaudit_value: >$pgaudit_value< indicates pgaudit is enabled"
     return 0
   else
+    echo "pgaudit_value: >$pgaudit_value< indicates pgaudit is NOT enabled"
     return 1
   fi
 }
@@ -102,17 +126,17 @@ case $current_parameter_group in
     echo "Uses default parameter group $current_parameter_group."
     echo "Creating new parameter group $db_instance and associating it with instance."
     create_and_associate_parameter_group "$current_parameter_group" "$db_instance"
-    reboot_instance "$db_instance"
+    reboot_instance "$db_instance" 0
     ;;
-  *$service_instance* )
-    echo "Parameter group already set to $service_instance_guid."
+  *$db_instance* )
+    echo "Parameter group already set to $db_instance."
     if pgaudit_is_enabled "$current_parameter_group"; then
       echo "pgaudit is already enabled in parameter group $current_parameter_group. Exiting."
       exit 0
     else
       echo "Enabling pgaudit in parameter group $current_parameter_group."
       setup_pgaudit_parameter "$current_parameter_group"
-      reboot_instance "$db_instance"
+      reboot_instance "$db_instance" 0
     fi
     exit 0
     ;;

From 4d599d23c00fdc017a4332dfa1317e529027d333 Mon Sep 17 00:00:00 2001
From: Peter Burkholder <peter.burkholder@gsa.gov>
Date: Wed, 31 Dec 2025 13:43:31 -0500
Subject: [PATCH 4/4] Try to capture reboot failure

---
 aws/rds/enable-pgaudit.sh | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/aws/rds/enable-pgaudit.sh b/aws/rds/enable-pgaudit.sh
index f1b52f41..294a0d59 100755
--- a/aws/rds/enable-pgaudit.sh
+++ b/aws/rds/enable-pgaudit.sh
@@ -1,24 +1,29 @@
 #!/bin/bash
 
+
 set -eu -o pipefail
 
+#org=cloud-gov-operators
+#space=peter.burkholder
+#service_instance=mia-django-db-x-psql15
+
+echo "Service Instance: $service_instance in Org: $org, Space: $space"
+
+
 function reboot_instance() {
   [ $# -ne 2 ] && echo "Function: reboot_instance <db_instance> <attempt>" && exit 1
   local db_instance=$1
   local attempt=$2
 
+  set +e
   aws rds reboot-db-instance \
     --db-instance-identifier "$db_instance"
-  
   status=$?
+  set -e
 
   if [ $status -eq 254 ]; then
     echo "Failed to reboot instance $db_instance status code 254"
-    if [ -z "${attempt+x}" ]; then
-      attempt=1
-    else
-      attempt=$((attempt + 1))
-    fi
+    attempt=$((attempt + 1))
     if [ $attempt -gt 5 ]; then
       echo "Exceeded maximum reboot attempts. Exiting."
       exit 1
@@ -84,15 +89,6 @@ function pgaudit_is_enabled() {
   fi
 }
 
-#org=epa-spds
-#space=test
-#service_instance=spds-db
-
-org=cloud-gov-operators
-space=peter.burkholder
-service_instance=mia-django-db-x-psql15
-
-echo "Service Instance: $service_instance in Org: $org, Space: $space"
 
 cf target -o $org -s $space > /dev/null
 service_instance_guid=$(cf service $service_instance --guid)