update to v0.6.2

check-prereqs

@@ -14,8 +14,12 @@ for arg in "$@"; do
   fi
 done
 
-# Source centralized color scheme
-source "$(dirname "$0")/set-term-colorscheme.sh"
+COLORSCHEME_SH="$(dirname "$0")/set-term-colorscheme.sh"
+if [ -f "$COLORSCHEME_SH" ]; then
+  source "$COLORSCHEME_SH"
+else
+  ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m'
+fi
 
 if ! git --version &> /dev/null ; then
   printf "${ERR}Git not installed.${NC} Not entirely sure how you got here without it, but to install see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git\n"
@@ -165,12 +169,75 @@ printf "\n"
 
 # Check Azure CLI installation
 AZCLI_REASON="Required if deploying to Azure or using Microsoft 365 data sources."
+AZ_ENTRA_ADMIN_ROLES_REASON="Microsoft 365 / Entra connectors require the authenticated principal to have Global Administrator or Application Administrator directory roles."
+AZ_ENTRA_ROLES_DOC="https://learn.microsoft.com/en-us/entra/identity/role-based-access-control/view-assignments"
 if ! az --version &> /dev/null ; then
   printf "${ERR}Azure CLI is not installed.${NC} ${AZCLI_REASON} See https://docs.microsoft.com/en-us/cli/azure/install-azure-cli\n"
   if $HOMEBREW_AVAILABLE; then printf " or, as you have Homebrew available, run ${CODE}brew install azure-cli${NC}\n"; fi
+  printf "\t- ${WARN}If you intend to use Microsoft 365 connectors, your Azure principal will still need ${CODE}Global Administrator${NC} or ${CODE}Application Administrator${NC} Entra directory roles. See ${AZ_ENTRA_ROLES_DOC}${NC}\n"
 else
   # how can pipe to sed or something to strip extra whitespace out?
   printf "Azure CLI version ${CODE}`az --version --only-show-errors | head -n 1`${NC} is installed.\n"
-  printf "\t- make sure ${CODE}az account show${NC} is the user/tenant you expect. If not, ${CODE}az login --allow-no-subscription${NC} to authenticate. $AZCLI_REASON\n"
+  printf "\t- make sure ${CODE}az account show${NC} is the user/tenant you expect. If not, ${CODE}az login --allow-no-subscriptions${NC} to authenticate. $AZCLI_REASON\n"
+
+  if az account show &> /dev/null; then
+    AZ_TENANT_ID=$(az account show --query tenantId -o tsv 2>/dev/null)
+    AZ_USER_NAME=$(az account show --query user.name -o tsv 2>/dev/null)
+    if [[ -n "$AZ_USER_NAME" ]]; then
+      printf "\t- signed in as ${CODE}${AZ_USER_NAME}${NC}"
+      if [[ -n "$AZ_TENANT_ID" ]]; then
+        printf " (tenant ${CODE}${AZ_TENANT_ID}${NC})"
+      fi
+      printf ".\n"
+    fi
+
+    AZ_GRAPH_ROLES_URL="https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.directoryRole"
+    AZ_SKIP_ROLE_CHECK=false
+    AZ_PRINCIPAL_TYPE=$(az account show --query user.type -o tsv 2>/dev/null)
+    if [[ "$AZ_PRINCIPAL_TYPE" == "servicePrincipal" ]]; then
+      AZ_SP_APP_ID=$(az account show --query user.name -o tsv 2>/dev/null)
+      AZ_SP_OBJECT_ID=$(az ad sp show --id "$AZ_SP_APP_ID" --query id -o tsv 2>/dev/null)
+      if [[ -z "$AZ_SP_OBJECT_ID" ]]; then
+        AZ_SKIP_ROLE_CHECK=true
+        printf "\t- ${WARN}Could not resolve service principal object ID for Entra directory role check.${NC}\n"
+        printf "\t  ${AZ_ENTRA_ADMIN_ROLES_REASON} Verify assignments in the Microsoft Entra admin center. See ${AZ_ENTRA_ROLES_DOC}\n"
+      else
+        AZ_GRAPH_ROLES_URL="https://graph.microsoft.com/v1.0/servicePrincipals/${AZ_SP_OBJECT_ID}/transitiveMemberOf/microsoft.graph.directoryRole"
+      fi
+    fi
+
+    if [[ "$AZ_SKIP_ROLE_CHECK" != "true" ]]; then
+      AZ_ROLE_NAMES=$(az rest --method GET --url "$AZ_GRAPH_ROLES_URL" --query "value[].displayName" -o tsv 2>/dev/null)
+      AZ_REST_EXIT=$?
+      if [[ $AZ_REST_EXIT -ne 0 ]]; then
+        printf "\t- ${WARN}Could not verify Entra directory roles (Microsoft Graph request failed).${NC}\n"
+        printf "\t  ${AZ_ENTRA_ADMIN_ROLES_REASON} Verify assignments in the Microsoft Entra admin center. See ${AZ_ENTRA_ROLES_DOC}\n"
+      else
+        AZ_HAS_REQUIRED_ROLE=false
+        while IFS= read -r AZ_ROLE_NAME; do
+          if [[ "$AZ_ROLE_NAME" == "Global Administrator" || "$AZ_ROLE_NAME" == "Application Administrator" ]]; then
+            AZ_HAS_REQUIRED_ROLE=true
+            break
+          fi
+        done <<< "$AZ_ROLE_NAMES"
+
+        if $AZ_HAS_REQUIRED_ROLE; then
+          printf "\t- ${SUCCESS}Entra directory roles: signed-in principal has Global Administrator or Application Administrator.${NC}\n"
+        else
+          AZ_ROLE_LIST="${AZ_ROLE_NAMES//$'\n'/, }"
+          if [[ -z "$AZ_ROLE_LIST" ]]; then
+            AZ_ROLE_LIST="(none detected)"
+          fi
+          printf "\t- ${WARN}Entra directory roles: signed-in principal does not appear to have Global Administrator or Application Administrator.${NC}\n"
+          printf "\t  ${AZ_ENTRA_ADMIN_ROLES_REASON}\n"
+          printf "\t  Directory roles detected for this principal: ${CODE}${AZ_ROLE_LIST}${NC}.\n"
+          printf "\t  See ${AZ_ENTRA_ROLES_DOC}\n"
+        fi
+      fi
+    fi
+  else
+    printf "\t- ${WARN}Azure CLI is not authenticated.${NC} Run ${CODE}az login --allow-no-subscriptions${NC} to authenticate.\n"
+    printf "\t  Once authenticated, re-run this script to verify Entra directory role requirements (${CODE}Global Administrator${NC} or ${CODE}Application Administrator${NC}).\n"
+  fi
 fi
 

google-workspace.tf

@@ -7,7 +7,7 @@ provider "google" {
 
 
 module "worklytics_connectors_google_workspace" {
-  source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.6.1"
+  source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.6.2"
 
   google_workspace_connector_settings = var.google_workspace_connector_settings
 

init

@@ -24,12 +24,21 @@ fi
 #   - within example directory, such as `infra/examples-dev/aws`:
 #     ../../../tools/init-example.sh ~/code/psoxy
 #
-#   to repeat:
-#     ../../../tools/reset-example.sh
+#   to repeat init from scratch (prompts to back up terraform.tfvars, etc. first):
+#     ./reset-example              # dev examples only (symlink to tools/reset-example.sh)
+#     ./init ~/code/psoxy
+#     ./reset-example --recover
+#
+#   backup only (no reset):
+#     ./reset-example --backup
 
-# colors
-# Source centralized color scheme
-source "$(dirname "$0")/set-term-colorscheme.sh"
+# colors (optional here: copied examples don't bundle this file until after terraform init)
+COLORSCHEME_SH="$(dirname "$0")/set-term-colorscheme.sh"
+if [ -f "$COLORSCHEME_SH" ]; then
+  source "$COLORSCHEME_SH"
+else
+  ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m'
+fi
 
 EXPLICIT_REPO_CLONE_DIR=$1
 

main.tf

@@ -20,7 +20,7 @@ terraform {
 
 # general cases
 module "worklytics_connectors" {
-  source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.6.1"
+  source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.6.2"
 
   enabled_connectors                       = var.enabled_connectors
   connector_settings                       = var.connector_settings
@@ -62,9 +62,17 @@ locals {
     {}
   )
 
+  custom_bulk_connectors_with_defaults = {
+    for k, v in var.custom_bulk_connectors : k => merge({
+      display_name              = coalesce(try(v.display_name, null), replace(k, "-", " "))
+      worklytics_connector_id   = coalesce(try(v.worklytics_connector_id, null), "bulk-import-psoxy")
+      worklytics_connector_name = coalesce(try(v.worklytics_connector_name, null), "${coalesce(try(v.display_name, null), replace(k, "-", " "))} via Psoxy")
+    }, v)
+  }
+
   bulk_connectors = merge(
     module.worklytics_connectors.enabled_bulk_connectors,
-    var.custom_bulk_connectors,
+    local.custom_bulk_connectors_with_defaults,
   )
 
   source_authorization_todos = concat(
@@ -111,7 +119,7 @@ locals {
 }
 
 module "psoxy" {
-  source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-host?ref=v0.6.1"
+  source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-host?ref=v0.6.2"
 
   environment_name                     = var.environment_name
   aws_account_id                       = var.aws_account_id
@@ -188,21 +196,23 @@ locals {
 module "connection_in_worklytics" {
   for_each = local.all_instances
 
-  source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-proxy-connection-aws?ref=v0.6.1"
+  source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-proxy-connection-aws?ref=v0.6.2"
 
   proxy_instance_id    = each.key
   worklytics_host      = var.worklytics_host
   aws_region           = var.aws_region
   aws_role_arn         = module.psoxy.caller_role_arn
   proxy_endpoint_url   = try(each.value.endpoint_url, null)
   bucket_name          = try(each.value.sanitized_bucket, null)
-  connector_id         = try(local.all_connectors[each.key].worklytics_connector_id, "")
-  display_name         = try(local.all_connectors[each.key].worklytics_connector_name, "${local.all_connectors[each.key].display_name} via Psoxy", "")
+  connector_id         = try(local.all_connectors[each.key].worklytics_connector_id, "bulk-import-psoxy", "")
+  display_name         = try(local.all_connectors[each.key].worklytics_connector_name, "${try(local.all_connectors[each.key].display_name, replace(each.key, "-", " "))} via Psoxy", "${replace(each.key, "-", " ")} via Psoxy")
   todo_step            = module.psoxy.next_todo_step
   todos_as_local_files = var.todos_as_local_files
 
-  connector_settings_to_provide = try(each.value.settings_to_provide, {})
-
+  connector_settings_to_provide = merge(
+    try(each.value.settings_to_provide, {}),
+    try(local.all_connectors[each.key].settings_to_provide, {}),
+  )
 }
 
 output "path_to_deployment_jar" {

msft-365.tf

@@ -1,7 +1,7 @@
 # BEGIN MSFT
 
 module "worklytics_connectors_msft_365" {
-  source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.6.1"
+  source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.6.2"
 
   msft_365_connector_settings = var.msft_365_connector_settings
 
@@ -50,7 +50,7 @@ data "aws_region" "current" {
 module "cognito_identity_pool" {
   count = local.msft_365_enabled ? 1 : 0 # only provision identity pool if MSFT-365 connectors are enabled
 
-  source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-pool?ref=v0.6.1"
+  source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-pool?ref=v0.6.2"
 
   developer_provider_name = local.developer_provider_name
   name                    = "${local.env_qualifier}-azure-ad-federation"
@@ -72,7 +72,7 @@ locals {
 module "cognito_identity" {
   count = local.msft_365_enabled ? 1 : 0 # only provision identity pool if MSFT-365 connectors are enabled
 
-  source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-identity-cli?ref=v0.6.1"
+  source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-identity-cli?ref=v0.6.2"
 
 
   aws_region       = data.aws_region.current.id
@@ -109,7 +109,7 @@ locals {
 module "msft_connection_auth_federation" {
   for_each = local.provision_entraid_apps ? local.enabled_to_entraid_object : local.shared_to_entraid_object
 
-  source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.6.1"
+  source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.6.2"
 
   application_id = each.value.connector_id
   display_name   = "${local.env_qualifier}AccessFromAWS"

variables.tf

@@ -319,12 +319,13 @@ variable "custom_bulk_connectors" {
     worklytics_connector_name = optional(string, "Custom Bulk Data via Psoxy")
     rules_file                = optional(string)
     rules = optional(object({
-      pseudonymFormat       = optional(string, "URL_SAFE_TOKEN")
-      columnsToRedact       = optional(list(string)) # columns to remove from CSV
-      columnsToInclude      = optional(list(string)) # if you prefer to include only an explicit list of columns, rather than redacting those you don't want
-      columnsToPseudonymize = optional(list(string)) # columns to pseudonymize
-      columnsToDuplicate    = optional(map(string))  # columns to create copy of; name --> new name
-      columnsToRename       = optional(map(string))  # columns to rename: original name --> new name; renames applied BEFORE pseudonymization
+      pseudonymFormat                = optional(string, "URL_SAFE_TOKEN")
+      columnsToRedact                = optional(list(string)) # columns to remove from CSV
+      columnsToInclude               = optional(list(string)) # if you prefer to include only an explicit list of columns, rather than redacting those you don't want
+      columnsToPseudonymize          = optional(list(string)) # columns to pseudonymize
+      columnsToPseudonymizeIfPresent = optional(list(string)) # columns to pseudonymize when present; omit if absent
+      columnsToDuplicate             = optional(map(string))  # columns to create copy of; name --> new name
+      columnsToRename                = optional(map(string))  # columns to rename: original name --> new name; renames applied BEFORE pseudonymization
       fieldsToTransform = optional(map(object({
         newName    = string
         transforms = optional(list(map(string)), [])