From e3b474d1083bab7b3c56ec8eca770c57a03c20d4 Mon Sep 17 00:00:00 2001 From: Erik Schultink Date: Fri, 29 May 2026 14:55:04 -0700 Subject: [PATCH] Update example to v0.6.2 --- check-prereqs | 73 +++++++++++++++++++++++++++++++++++++++++++-- google-workspace.tf | 2 +- init | 19 ++++++++---- main.tf | 26 +++++++++++----- msft-365.tf | 8 ++--- reset-example | 64 --------------------------------------- variables.tf | 13 ++++---- 7 files changed, 114 insertions(+), 91 deletions(-) delete mode 100755 reset-example diff --git a/check-prereqs b/check-prereqs index 32886a4..9e1528f 100755 --- a/check-prereqs +++ b/check-prereqs @@ -14,8 +14,12 @@ for arg in "$@"; do fi done -# Source centralized color scheme -source "$(dirname "$0")/set-term-colorscheme.sh" +COLORSCHEME_SH="$(dirname "$0")/set-term-colorscheme.sh" +if [ -f "$COLORSCHEME_SH" ]; then + source "$COLORSCHEME_SH" +else + ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m' +fi if ! git --version &> /dev/null ; then printf "${ERR}Git not installed.${NC} Not entirely sure how you got here without it, but to install see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git\n" @@ -165,12 +169,75 @@ printf "\n" # Check Azure CLI installation AZCLI_REASON="Required if deploying to Azure or using Microsoft 365 data sources." +AZ_ENTRA_ADMIN_ROLES_REASON="Microsoft 365 / Entra connectors require the authenticated principal to have Global Administrator or Application Administrator directory roles." +AZ_ENTRA_ROLES_DOC="https://learn.microsoft.com/en-us/entra/identity/role-based-access-control/view-assignments" if ! az --version &> /dev/null ; then printf "${ERR}Azure CLI is not installed.${NC} ${AZCLI_REASON} See https://docs.microsoft.com/en-us/cli/azure/install-azure-cli\n" if $HOMEBREW_AVAILABLE; then printf " or, as you have Homebrew available, run ${CODE}brew install azure-cli${NC}\n"; fi + printf "\t- ${WARN}If you intend to use Microsoft 365 connectors, your Azure principal will still need ${CODE}Global Administrator${NC} or ${CODE}Application Administrator${NC} Entra directory roles. See ${AZ_ENTRA_ROLES_DOC}${NC}\n" else # how can pipe to sed or something to strip extra whitespace out? printf "Azure CLI version ${CODE}`az --version --only-show-errors | head -n 1`${NC} is installed.\n" - printf "\t- make sure ${CODE}az account show${NC} is the user/tenant you expect. If not, ${CODE}az login --allow-no-subscription${NC} to authenticate. $AZCLI_REASON\n" + printf "\t- make sure ${CODE}az account show${NC} is the user/tenant you expect. If not, ${CODE}az login --allow-no-subscriptions${NC} to authenticate. $AZCLI_REASON\n" + + if az account show &> /dev/null; then + AZ_TENANT_ID=$(az account show --query tenantId -o tsv 2>/dev/null) + AZ_USER_NAME=$(az account show --query user.name -o tsv 2>/dev/null) + if [[ -n "$AZ_USER_NAME" ]]; then + printf "\t- signed in as ${CODE}${AZ_USER_NAME}${NC}" + if [[ -n "$AZ_TENANT_ID" ]]; then + printf " (tenant ${CODE}${AZ_TENANT_ID}${NC})" + fi + printf ".\n" + fi + + AZ_GRAPH_ROLES_URL="https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.directoryRole" + AZ_SKIP_ROLE_CHECK=false + AZ_PRINCIPAL_TYPE=$(az account show --query user.type -o tsv 2>/dev/null) + if [[ "$AZ_PRINCIPAL_TYPE" == "servicePrincipal" ]]; then + AZ_SP_APP_ID=$(az account show --query user.name -o tsv 2>/dev/null) + AZ_SP_OBJECT_ID=$(az ad sp show --id "$AZ_SP_APP_ID" --query id -o tsv 2>/dev/null) + if [[ -z "$AZ_SP_OBJECT_ID" ]]; then + AZ_SKIP_ROLE_CHECK=true + printf "\t- ${WARN}Could not resolve service principal object ID for Entra directory role check.${NC}\n" + printf "\t ${AZ_ENTRA_ADMIN_ROLES_REASON} Verify assignments in the Microsoft Entra admin center. See ${AZ_ENTRA_ROLES_DOC}\n" + else + AZ_GRAPH_ROLES_URL="https://graph.microsoft.com/v1.0/servicePrincipals/${AZ_SP_OBJECT_ID}/transitiveMemberOf/microsoft.graph.directoryRole" + fi + fi + + if [[ "$AZ_SKIP_ROLE_CHECK" != "true" ]]; then + AZ_ROLE_NAMES=$(az rest --method GET --url "$AZ_GRAPH_ROLES_URL" --query "value[].displayName" -o tsv 2>/dev/null) + AZ_REST_EXIT=$? + if [[ $AZ_REST_EXIT -ne 0 ]]; then + printf "\t- ${WARN}Could not verify Entra directory roles (Microsoft Graph request failed).${NC}\n" + printf "\t ${AZ_ENTRA_ADMIN_ROLES_REASON} Verify assignments in the Microsoft Entra admin center. See ${AZ_ENTRA_ROLES_DOC}\n" + else + AZ_HAS_REQUIRED_ROLE=false + while IFS= read -r AZ_ROLE_NAME; do + if [[ "$AZ_ROLE_NAME" == "Global Administrator" || "$AZ_ROLE_NAME" == "Application Administrator" ]]; then + AZ_HAS_REQUIRED_ROLE=true + break + fi + done <<< "$AZ_ROLE_NAMES" + + if $AZ_HAS_REQUIRED_ROLE; then + printf "\t- ${SUCCESS}Entra directory roles: signed-in principal has Global Administrator or Application Administrator.${NC}\n" + else + AZ_ROLE_LIST="${AZ_ROLE_NAMES//$'\n'/, }" + if [[ -z "$AZ_ROLE_LIST" ]]; then + AZ_ROLE_LIST="(none detected)" + fi + printf "\t- ${WARN}Entra directory roles: signed-in principal does not appear to have Global Administrator or Application Administrator.${NC}\n" + printf "\t ${AZ_ENTRA_ADMIN_ROLES_REASON}\n" + printf "\t Directory roles detected for this principal: ${CODE}${AZ_ROLE_LIST}${NC}.\n" + printf "\t See ${AZ_ENTRA_ROLES_DOC}\n" + fi + fi + fi + else + printf "\t- ${WARN}Azure CLI is not authenticated.${NC} Run ${CODE}az login --allow-no-subscriptions${NC} to authenticate.\n" + printf "\t Once authenticated, re-run this script to verify Entra directory role requirements (${CODE}Global Administrator${NC} or ${CODE}Application Administrator${NC}).\n" + fi fi diff --git a/google-workspace.tf b/google-workspace.tf index 9def5b2..1a6af88 100644 --- a/google-workspace.tf +++ b/google-workspace.tf @@ -7,7 +7,7 @@ provider "google" { module "worklytics_connectors_google_workspace" { - source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.6.1" + source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-google-workspace?ref=v0.6.2" google_workspace_connector_settings = var.google_workspace_connector_settings diff --git a/init b/init index c82feb3..6a54396 100755 --- a/init +++ b/init @@ -24,12 +24,21 @@ fi # - within example directory, such as `infra/examples-dev/aws`: # ../../../tools/init-example.sh ~/code/psoxy # -# to repeat: -# ../../../tools/reset-example.sh +# to repeat init from scratch (prompts to back up terraform.tfvars, etc. first): +# ./reset-example # dev examples only (symlink to tools/reset-example.sh) +# ./init ~/code/psoxy +# ./reset-example --recover +# +# backup only (no reset): +# ./reset-example --backup -# colors -# Source centralized color scheme -source "$(dirname "$0")/set-term-colorscheme.sh" +# colors (optional here: copied examples don't bundle this file until after terraform init) +COLORSCHEME_SH="$(dirname "$0")/set-term-colorscheme.sh" +if [ -f "$COLORSCHEME_SH" ]; then + source "$COLORSCHEME_SH" +else + ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m' +fi EXPLICIT_REPO_CLONE_DIR=$1 diff --git a/main.tf b/main.tf index b51aa0b..b797c59 100644 --- a/main.tf +++ b/main.tf @@ -20,7 +20,7 @@ terraform { # general cases module "worklytics_connectors" { - source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.6.1" + source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors?ref=v0.6.2" enabled_connectors = var.enabled_connectors connector_settings = var.connector_settings @@ -62,9 +62,17 @@ locals { {} ) + custom_bulk_connectors_with_defaults = { + for k, v in var.custom_bulk_connectors : k => merge({ + display_name = coalesce(try(v.display_name, null), replace(k, "-", " ")) + worklytics_connector_id = coalesce(try(v.worklytics_connector_id, null), "bulk-import-psoxy") + worklytics_connector_name = coalesce(try(v.worklytics_connector_name, null), "${coalesce(try(v.display_name, null), replace(k, "-", " "))} via Psoxy") + }, v) + } + bulk_connectors = merge( module.worklytics_connectors.enabled_bulk_connectors, - var.custom_bulk_connectors, + local.custom_bulk_connectors_with_defaults, ) source_authorization_todos = concat( @@ -111,7 +119,7 @@ locals { } module "psoxy" { - source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-host?ref=v0.6.1" + source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-host?ref=v0.6.2" environment_name = var.environment_name aws_account_id = var.aws_account_id @@ -188,7 +196,7 @@ locals { module "connection_in_worklytics" { for_each = local.all_instances - source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-proxy-connection-aws?ref=v0.6.1" + source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-proxy-connection-aws?ref=v0.6.2" proxy_instance_id = each.key worklytics_host = var.worklytics_host @@ -196,13 +204,15 @@ module "connection_in_worklytics" { aws_role_arn = module.psoxy.caller_role_arn proxy_endpoint_url = try(each.value.endpoint_url, null) bucket_name = try(each.value.sanitized_bucket, null) - connector_id = try(local.all_connectors[each.key].worklytics_connector_id, "") - display_name = try(local.all_connectors[each.key].worklytics_connector_name, "${local.all_connectors[each.key].display_name} via Psoxy", "") + connector_id = try(local.all_connectors[each.key].worklytics_connector_id, "bulk-import-psoxy", "") + display_name = try(local.all_connectors[each.key].worklytics_connector_name, "${try(local.all_connectors[each.key].display_name, replace(each.key, "-", " "))} via Psoxy", "${replace(each.key, "-", " ")} via Psoxy") todo_step = module.psoxy.next_todo_step todos_as_local_files = var.todos_as_local_files - connector_settings_to_provide = try(each.value.settings_to_provide, {}) - + connector_settings_to_provide = merge( + try(each.value.settings_to_provide, {}), + try(local.all_connectors[each.key].settings_to_provide, {}), + ) } output "path_to_deployment_jar" { diff --git a/msft-365.tf b/msft-365.tf index 570f230..2fcf1ac 100644 --- a/msft-365.tf +++ b/msft-365.tf @@ -1,7 +1,7 @@ # BEGIN MSFT module "worklytics_connectors_msft_365" { - source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.6.1" + source = "git::https://github.com/worklytics/psoxy//infra/modules/worklytics-connectors-msft-365?ref=v0.6.2" msft_365_connector_settings = var.msft_365_connector_settings @@ -50,7 +50,7 @@ data "aws_region" "current" { module "cognito_identity_pool" { count = local.msft_365_enabled ? 1 : 0 # only provision identity pool if MSFT-365 connectors are enabled - source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-pool?ref=v0.6.1" + source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-pool?ref=v0.6.2" developer_provider_name = local.developer_provider_name name = "${local.env_qualifier}-azure-ad-federation" @@ -72,7 +72,7 @@ locals { module "cognito_identity" { count = local.msft_365_enabled ? 1 : 0 # only provision identity pool if MSFT-365 connectors are enabled - source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-identity-cli?ref=v0.6.1" + source = "git::https://github.com/worklytics/psoxy//infra/modules/aws-cognito-identity-cli?ref=v0.6.2" aws_region = data.aws_region.current.id @@ -109,7 +109,7 @@ locals { module "msft_connection_auth_federation" { for_each = local.provision_entraid_apps ? local.enabled_to_entraid_object : local.shared_to_entraid_object - source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.6.1" + source = "git::https://github.com/worklytics/psoxy//infra/modules/azuread-federated-credentials?ref=v0.6.2" application_id = each.value.connector_id display_name = "${local.env_qualifier}AccessFromAWS" diff --git a/reset-example b/reset-example deleted file mode 100755 index 5ad0935..0000000 --- a/reset-example +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# colors -COLORSCHEME_SH="$(dirname "$0")/set-term-colorscheme.sh" -if [ -f "$COLORSCHEME_SH" ]; then - source "$COLORSCHEME_SH" -else - ERR='\033[0;31m'; SUCCESS='\033[0;32m'; WARN='\033[1;33m'; INFO='\033[0;34m'; CODE='\033[0;36m'; NC='\033[0m' -fi - -# warn user that will delete a bunch of files -printf "This script will ${ERR}delete${NC} the your local terraform state, variable files, etc, to " -printf "reset to example template prior to ${INFO}./init${NC} and any terraform init/plan/apply you've done.\n" -printf "If you have ${ERR}NOT${NC} committed these files and/or your local changes, they will be lost.\n" -printf "Do you want to continue? (y/N): " -read -r response -if [[ ! "$response" =~ ^([yY][eE][sS]|[yY])$ ]]; then - printf "Exiting...\n" - exit 0 -fi - -# resets example to state prior to `./init` -rm .terraform.lock.hcl 2>/dev/null -rm build 2>/dev/null -rm update-bundle 2>/dev/null -rm psoxy-* 2>/dev/null -rm -rf .terraform 2>/dev/null -rm terraform.tfvars 2>/dev/null -rm terraform.tfstate 2>/dev/null - -# restore main.tf, if modified -printf "Restoring ${INFO}main.tf${NC} configuration file ...\n" -git checkout HEAD -- main.tf - -# check source-specific files that may have been deleted -FILES=("msft-365.tf" "msft-365-variables.tf" "google-workspace.tf" "google-workspace-variables.tf") - -check_and_restore_file() { - local file="$1" - - # Check the git status to find out if the file was deleted - if git status --short | grep -q "^ D $file"; then - # The file is deleted, restore it from the HEAD - printf "Configuration file ${INFO}$file${NC} was deleted, restoring...\n" - git checkout HEAD -- "$file" - - if [ $? -eq 0 ]; then - printf "${INFO}$file${NC} has been successfully restored.\n" - else - printf "${ERR}Error occurred while restoring '$file'${NC}\n" - return 1 - fi - fi -} - -# Loop through the files and pass each one to the check_and_restore_file function -for file in "${FILES[@]}"; do - check_and_restore_file "$file" -done - -if [[ -f upgrade-terraform-modules ]]; then - rm upgrade-terraform-modules -fi - diff --git a/variables.tf b/variables.tf index 7f33d14..3ea485c 100644 --- a/variables.tf +++ b/variables.tf @@ -319,12 +319,13 @@ variable "custom_bulk_connectors" { worklytics_connector_name = optional(string, "Custom Bulk Data via Psoxy") rules_file = optional(string) rules = optional(object({ - pseudonymFormat = optional(string, "URL_SAFE_TOKEN") - columnsToRedact = optional(list(string)) # columns to remove from CSV - columnsToInclude = optional(list(string)) # if you prefer to include only an explicit list of columns, rather than redacting those you don't want - columnsToPseudonymize = optional(list(string)) # columns to pseudonymize - columnsToDuplicate = optional(map(string)) # columns to create copy of; name --> new name - columnsToRename = optional(map(string)) # columns to rename: original name --> new name; renames applied BEFORE pseudonymization + pseudonymFormat = optional(string, "URL_SAFE_TOKEN") + columnsToRedact = optional(list(string)) # columns to remove from CSV + columnsToInclude = optional(list(string)) # if you prefer to include only an explicit list of columns, rather than redacting those you don't want + columnsToPseudonymize = optional(list(string)) # columns to pseudonymize + columnsToPseudonymizeIfPresent = optional(list(string)) # columns to pseudonymize when present; omit if absent + columnsToDuplicate = optional(map(string)) # columns to create copy of; name --> new name + columnsToRename = optional(map(string)) # columns to rename: original name --> new name; renames applied BEFORE pseudonymization fieldsToTransform = optional(map(object({ newName = string transforms = optional(list(map(string)), [])