diff --git a/CHANGELOG.md b/CHANGELOG.md index fab5508c9..679fdb837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ in each release's notes. Changes to be including in future/planned release notes will be added here. +## [Unreleased] +- `aws`/`gcp`: fix Terraform plan failure when `enable_remote_resources = true` but no artifacts bucket exists (e.g. with a prebuilt `deployment_bundle`). When remote resources are enabled, an artifacts bucket is now provisioned if one is not already created or provided via `artifacts_bucket_name` / `custom_artifacts_bucket_name`. + ## [0.6.5](https://github.com/Worklytics/psoxy/releases/tag/v0.6.5) - added `claude-enterprise-analytics` connector in **beta**; imports per-user daily activity, token usage, and cost data from the [Claude Enterprise Analytics API](https://support.claude.com/en/articles/13703965-claude-enterprise-analytics-api-reference-guide); see [docs/sources/anthropic/claude-enterprise-analytics/README.md](docs/sources/anthropic/claude-enterprise-analytics/README.md) diff --git a/docs/configuration/remote-resources.md b/docs/configuration/remote-resources.md index 36f11a9be..b53514a57 100644 --- a/docs/configuration/remote-resources.md +++ b/docs/configuration/remote-resources.md @@ -29,12 +29,17 @@ mounted locally. ## Terraform Configuration -By default, the host modules in this repository (`aws-host` and `gcp-host`) will configure the -`REMOTE_RESOURCE_BUCKET` for you if you set the `enable_remote_resources` variable to `true`. This -automatically wires the **artifacts bucket** (used for deployment bundles) as the remote resource bucket. +Remote resources are **opt-in**. Set `enable_remote_resources = true` on the host module +(`aws-host` or `gcp-host`) when you want psoxy to load rules, NLP models, or other assets from +the artifacts bucket at runtime. The host module does not infer this from your connector list. + +When enabled, the host module uses the artifacts bucket — either one you provide +(`artifacts_bucket_name` / `custom_artifacts_bucket_name`), one already provisioned for a local +deployment bundle, or a newly provisioned bucket when using a prebuilt `s3://` / `gs://` +`deployment_bundle`. > [!IMPORTANT] -> If you configure an existing bucket (e.g., by providing `artifacts_bucket_name`), the bucket must already exist. +> If you supply an existing bucket (`artifacts_bucket_name` / `custom_artifacts_bucket_name`), it must already exist. > > The Terraform runner (the credentials running the `terraform` command) must have sufficient IAM permissions on that bucket to apply permissions, since it will grant read access to the proxy's service account or Lambda execution role. @@ -46,7 +51,6 @@ module "psoxy" { # ... existing configuration ... - # Enable remote resource loading from the artifacts S3 bucket enable_remote_resources = true } ``` @@ -63,7 +67,6 @@ module "psoxy" { # ... existing configuration ... - # Enable remote resource loading from the artifacts GCS bucket enable_remote_resources = true } ``` diff --git a/infra/modules/aws-host/main.tf b/infra/modules/aws-host/main.tf index 888401a90..3b3bfa31a 100644 --- a/infra/modules/aws-host/main.tf +++ b/infra/modules/aws-host/main.tf @@ -110,6 +110,7 @@ module "psoxy" { enable_webhook_testing = local.enable_webhook_testing webhook_allow_origins = distinct(flatten([for v in var.webhook_collectors : v.allow_origins])) artifacts_bucket_name = var.artifacts_bucket_name + enable_remote_resources = var.enable_remote_resources allowed_data_access_ip_blocks = var.allowed_data_access_ip_blocks allowed_webhook_ip_blocks = var.allowed_webhook_ip_blocks } @@ -127,6 +128,7 @@ locals { connector_instance_resource_path = { for k, v in merge(var.api_connectors, var.bulk_connectors, var.webhook_collectors) : k => "${local.shared_resource_path}${replace(upper(k), "-", "_")}/" } + remote_resources_enabled = var.enable_remote_resources && module.psoxy.artifacts_bucket_name != null # convert custom_side_outputs to the format expected by the psoxy module custom_original_side_outputs = { for k, v in var.custom_side_outputs : @@ -261,9 +263,9 @@ module "api_connector" { var.general_environment_variables, ) - remote_resource_bucket = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null - remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null - remote_resource_shared_path = var.enable_remote_resources ? local.shared_resource_path : null + remote_resource_bucket = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null + remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null + remote_resource_shared_path = local.remote_resources_enabled ? local.shared_resource_path : null } @@ -349,9 +351,9 @@ module "bulk_connector" { var.general_environment_variables ) - remote_resource_bucket = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null - remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null - remote_resource_shared_path = var.enable_remote_resources ? local.shared_resource_path : null + remote_resource_bucket = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null + remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null + remote_resource_shared_path = local.remote_resources_enabled ? local.shared_resource_path : null } @@ -401,9 +403,9 @@ module "webhook_collectors" { var.general_environment_variables, ) - remote_resource_bucket = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null - remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null - remote_resource_shared_path = var.enable_remote_resources ? local.shared_resource_path : null + remote_resource_bucket = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null + remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null + remote_resource_shared_path = local.remote_resources_enabled ? local.shared_resource_path : null } # Policy to allow test caller to invoke webhook collector urls and sign webhook requests diff --git a/infra/modules/aws-host/variables.tf b/infra/modules/aws-host/variables.tf index d9c9e1d5b..09763b309 100644 --- a/infra/modules/aws-host/variables.tf +++ b/infra/modules/aws-host/variables.tf @@ -468,13 +468,12 @@ variable "todo_step" { variable "artifacts_bucket_name" { type = string - description = "Name of an existing S3 bucket to use for deployment artifacts. If null, one will be provisioned if needed." + description = "Name of an existing S3 bucket to use for deployment artifacts and remote resources (rules, NLP models, etc.). If null, one will be provisioned when needed for a local deployment bundle or when enable_remote_resources is true." default = null } - variable "enable_remote_resources" { type = bool - description = "**beta** Whether to enable remote resource loading from the artifacts S3 bucket (rules, NLP models, etc.). When true, sets REMOTE_RESOURCE_BUCKET env var and grants s3:GetObject to each Lambda. Default will change to `true` in next major version." - default = false # will change to true in 0.7.x + description = "**beta** Whether to enable remote resource loading from the artifacts S3 bucket (rules, NLP models, etc.). When true, sets REMOTE_RESOURCE_BUCKET env var and grants s3:GetObject to each Lambda. Provisions an artifacts bucket if one is not already created or provided." + default = false } diff --git a/infra/modules/aws-proxy-lambda/main.tf b/infra/modules/aws-proxy-lambda/main.tf index f13d52efb..876f8d0e0 100644 --- a/infra/modules/aws-proxy-lambda/main.tf +++ b/infra/modules/aws-proxy-lambda/main.tf @@ -98,8 +98,8 @@ resource "aws_lambda_function" "instance" { length(var.path_to_shared_ssm_parameters) > 0 ? { PATH_TO_SHARED_CONFIG = var.path_to_shared_ssm_parameters } : {}, local.is_instance_ssm_prefix_default ? {} : { PATH_TO_INSTANCE_CONFIG = var.path_to_instance_ssm_parameters }, var.remote_resource_bucket != null ? { REMOTE_RESOURCE_BUCKET = var.remote_resource_bucket } : {}, - var.remote_resource_instance_path != null ? { INSTANCE_RESOURCE_PATH = var.remote_resource_instance_path } : {}, - var.remote_resource_shared_path != null ? { SHARED_RESOURCE_PATH = var.remote_resource_shared_path } : {}, + var.remote_resource_bucket != null && var.remote_resource_instance_path != null ? { INSTANCE_RESOURCE_PATH = var.remote_resource_instance_path } : {}, + var.remote_resource_bucket != null && var.remote_resource_shared_path != null ? { SHARED_RESOURCE_PATH = var.remote_resource_shared_path } : {}, ) } @@ -301,14 +301,14 @@ locals { remote_resource_instance_prefix = var.remote_resource_instance_path != null ? trimsuffix(var.remote_resource_instance_path, "/") : "" remote_resource_shared_prefix = var.remote_resource_shared_path != null ? trimsuffix(var.remote_resource_shared_path, "/") : "" - remote_resource_s3_object_arns = distinct(compact([ + remote_resource_s3_object_arns = var.remote_resource_bucket != null ? distinct(compact([ var.remote_resource_instance_path != null ? ( local.remote_resource_instance_prefix != "" ? "arn:aws:s3:::${var.remote_resource_bucket}/${local.remote_resource_instance_prefix}/*" : "arn:aws:s3:::${var.remote_resource_bucket}/*" ) : "", var.remote_resource_shared_path != null ? ( local.remote_resource_shared_prefix != "" ? "arn:aws:s3:::${var.remote_resource_bucket}/${local.remote_resource_shared_prefix}/*" : "arn:aws:s3:::${var.remote_resource_bucket}/*" ) : "", - ])) + ])) : [] remote_resource_bucket_statements = var.remote_resource_bucket != null ? [{ Sid = "ReadRemoteResourceBucket" diff --git a/infra/modules/aws-proxy-lambda/remote_resource_iam.tftest.hcl b/infra/modules/aws-proxy-lambda/remote_resource_iam.tftest.hcl index 57a4df55c..7372d651f 100644 --- a/infra/modules/aws-proxy-lambda/remote_resource_iam.tftest.hcl +++ b/infra/modules/aws-proxy-lambda/remote_resource_iam.tftest.hcl @@ -93,3 +93,18 @@ run "duplicate_prefixes_dedupe_to_one_arn" { condition = toset(one([for s in jsondecode(aws_iam_policy.required_resource_access.policy).Statement : s.Resource if s.Sid == "ReadRemoteResourceBucket"])) == toset(["arn:aws:s3:::my-artifacts/shared/*"]) } } + +run "paths_without_bucket_skip_iam" { + command = plan + + variables { + remote_resource_bucket = null + remote_resource_instance_path = "instances/foo/" + remote_resource_shared_path = "shared/models/" + } + + assert { + error_message = "paths without a bucket should not create ReadRemoteResourceBucket IAM statement" + condition = length([for s in jsondecode(aws_iam_policy.required_resource_access.policy).Statement : s if s.Sid == "ReadRemoteResourceBucket"]) == 0 + } +} diff --git a/infra/modules/aws-webhook-collector/variables.tf b/infra/modules/aws-webhook-collector/variables.tf index bd3a0cd07..6998bb28b 100644 --- a/infra/modules/aws-webhook-collector/variables.tf +++ b/infra/modules/aws-webhook-collector/variables.tf @@ -201,11 +201,9 @@ variable "provision_auth_key" { validation { condition = ( - var.provision_auth_key == null || - ( - try(var.provision_auth_key.rotation_days, null) == null || - try(var.provision_auth_key.rotation_days, 0) > 0 - ) + var.provision_auth_key == null ? true : + var.provision_auth_key.rotation_days == null ? true : + var.provision_auth_key.rotation_days > 0 ) error_message = "If `provision_auth_key` is provided, `rotation_days` must be a positive number or null." } diff --git a/infra/modules/aws/main.tf b/infra/modules/aws/main.tf index 216bb367b..36b8c4a09 100644 --- a/infra/modules/aws/main.tf +++ b/infra/modules/aws/main.tf @@ -158,8 +158,8 @@ module "psoxy_package" { locals { # determine if the JAR is local and should be uploaded directly from plan-time variables is_local_jar = var.deployment_bundle == null || !startswith(coalesce(var.deployment_bundle, "unknown"), "s3://") - should_provision_bucket = local.is_local_jar && var.artifacts_bucket_name == null - target_artifacts_bucket = var.artifacts_bucket_name != null ? var.artifacts_bucket_name : (local.should_provision_bucket ? aws_s3_bucket.artifacts[0].bucket : null) + should_provision_bucket = var.artifacts_bucket_name == null && (local.is_local_jar || var.enable_remote_resources) + target_artifacts_bucket = coalesce(var.artifacts_bucket_name, try(aws_s3_bucket.artifacts[0].bucket, null)) should_upload_object = local.is_local_jar && (var.artifacts_bucket_name != null || local.should_provision_bucket) } diff --git a/infra/modules/aws/variables.tf b/infra/modules/aws/variables.tf index 5ea5e4c53..830ac8596 100644 --- a/infra/modules/aws/variables.tf +++ b/infra/modules/aws/variables.tf @@ -148,10 +148,16 @@ variable "enable_webhook_testing" { variable "artifacts_bucket_name" { type = string - description = "Name of an existing S3 bucket to use for deployment artifacts. If null, one will be provisioned if needed." + description = "Name of an existing S3 bucket to use for deployment artifacts and remote resources (rules, NLP models, etc.). If null, one will be provisioned when needed for a local deployment bundle or when enable_remote_resources is true." default = null } +variable "enable_remote_resources" { + type = bool + description = "Whether to provision an artifacts bucket for remote resources when one is not otherwise needed for deployment (e.g. with an s3:// deployment_bundle)." + default = false +} + variable "allowed_data_access_ip_blocks" { description = <<-EOT IPs or CIDR blocks allowed to make data access requests. When non-empty, adds infrastructure-level aws:SourceIp conditions on api-caller role assume-role policies (see docs/configuration/ip-allowlisting.md). Application-layer enforcement is configured separately on proxy Lambdas via the host module. diff --git a/infra/modules/gcp-host/main.tf b/infra/modules/gcp-host/main.tf index 701d1e37b..a2996e6e9 100644 --- a/infra/modules/gcp-host/main.tf +++ b/infra/modules/gcp-host/main.tf @@ -26,6 +26,7 @@ locals { connector_instance_resource_path = { for k, v in merge(var.api_connectors, var.bulk_connectors, var.webhook_collectors) : k => "${local.shared_resource_path}${replace(upper(k), "-", "_")}/" } + remote_resources_enabled = var.enable_remote_resources && module.psoxy.artifacts_bucket_name != null # rules_file paths may be absolute, relative to the Terraform root module (deployment dir), or # relative to psoxy_base_dir (paths into the psoxy repo, eg docs/sources/...) @@ -99,6 +100,7 @@ module "psoxy" { provision_testing_infra = var.provision_testing_infra gcp_principals_authorized_to_test = var.gcp_principals_authorized_to_test custom_artifacts_bucket_name = var.custom_artifacts_bucket_name + enable_remote_resources = var.enable_remote_resources support_bulk_mode = length(var.bulk_connectors) > 0 support_webhook_collectors = length(var.webhook_collectors) > 0 vpc_config = var.vpc_config @@ -237,7 +239,7 @@ module "api_connector" { environment_id_prefix = local.environment_id_prefix instance_id = each.key service_account_email = google_service_account.api_connectors[each.key].email - artifacts_bucket_name = module.psoxy.artifacts_bucket_name + artifacts_bucket_name = module.psoxy.deployment_bundle_bucket deployment_bundle_object_name = module.psoxy.deployment_bundle_object_name artifact_repository_id = module.psoxy.artifact_repository vpc_config = module.psoxy.vpc_config @@ -282,9 +284,9 @@ module "api_connector" { var.general_environment_variables, ) - remote_resource_bucket = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null - remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null - remote_resource_shared_path = var.enable_remote_resources ? local.shared_resource_path : null + remote_resource_bucket = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null + remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null + remote_resource_shared_path = local.remote_resources_enabled ? local.shared_resource_path : null secret_bindings = merge( local.secrets_bound_as_env_vars[each.key], @@ -341,7 +343,7 @@ module "webhook_collector" { service_account_id = google_service_account.webhook_collector[each.key].id email = google_service_account.webhook_collector[each.key].email } - artifacts_bucket_name = module.psoxy.artifacts_bucket_name + artifacts_bucket_name = module.psoxy.deployment_bundle_bucket deployment_bundle_object_name = module.psoxy.deployment_bundle_object_name artifact_repository_id = module.psoxy.artifact_repository path_to_repo_root = var.psoxy_base_dir @@ -378,9 +380,9 @@ module "webhook_collector" { var.general_environment_variables, ) - remote_resource_bucket = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null - remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null - remote_resource_shared_path = var.enable_remote_resources ? local.shared_resource_path : null + remote_resource_bucket = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null + remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null + remote_resource_shared_path = local.remote_resources_enabled ? local.shared_resource_path : null secret_bindings = module.psoxy.secrets @@ -404,7 +406,7 @@ module "bulk_connector" { worklytics_sa_emails = var.worklytics_sa_emails config_parameter_prefix = local.config_parameter_prefix source_kind = each.value.source_kind - artifacts_bucket_name = module.psoxy.artifacts_bucket_name + artifacts_bucket_name = module.psoxy.deployment_bundle_bucket artifact_repository_id = module.psoxy.artifact_repository deployment_bundle_object_name = module.psoxy.deployment_bundle_object_name psoxy_base_dir = var.psoxy_base_dir @@ -446,9 +448,9 @@ module "bulk_connector" { var.general_environment_variables, ) - remote_resource_bucket = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null - remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null - remote_resource_shared_path = var.enable_remote_resources ? local.shared_resource_path : null + remote_resource_bucket = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null + remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null + remote_resource_shared_path = local.remote_resources_enabled ? local.shared_resource_path : null depends_on = [ module.psoxy # some of the set-up IAM grants done there, but not EXPLICITLY passed out as outputs and into above as inputs, are required; so make this explicit diff --git a/infra/modules/gcp-host/variables.tf b/infra/modules/gcp-host/variables.tf index a5a65cff0..7fdb6e3d6 100644 --- a/infra/modules/gcp-host/variables.tf +++ b/infra/modules/gcp-host/variables.tf @@ -181,7 +181,7 @@ variable "kms_key_ring" { variable "custom_artifacts_bucket_name" { type = string - description = "name of bucket to use for custom artifacts, if you want something other than default" + description = "Name of an existing GCS bucket to use for deployment artifacts and remote resources (rules, NLP models, etc.). If null, one will be provisioned when needed for a local deployment bundle or when enable_remote_resources is true." default = null } @@ -463,6 +463,6 @@ variable "max_instances_per_api_connector" { variable "enable_remote_resources" { type = bool - description = "**beta** Whether to enable remote resource loading from the artifacts GCS bucket (rules, NLP models, etc.). When true, sets REMOTE_RESOURCE_BUCKET env var and grants roles/storage.objectViewer to each Cloud Function. Default will change to `true` in next major version." - default = false # will change to true in 0.7.x + description = "**beta** Whether to enable remote resource loading from the artifacts GCS bucket (rules, NLP models, etc.). When true, sets REMOTE_RESOURCE_BUCKET env var and grants roles/storage.objectViewer to each Cloud Function. Provisions an artifacts bucket if one is not already created or provided." + default = false } diff --git a/infra/modules/gcp/main.tf b/infra/modules/gcp/main.tf index b5b208505..e141d4bad 100644 --- a/infra/modules/gcp/main.tf +++ b/infra/modules/gcp/main.tf @@ -201,9 +201,10 @@ module "psoxy_package" { locals { # NOTE: `try` needed here bc Terraform doesn't short-circuit boolean evaluation - is_remote_bundle = var.deployment_bundle != null && try(startswith(var.deployment_bundle, "gs://"), false) - remote_bucket_name = local.is_remote_bundle ? split("/", var.deployment_bundle)[2] : null - remote_bundle_artifact = local.is_remote_bundle ? split("/", var.deployment_bundle)[3] : null + is_remote_bundle = var.deployment_bundle != null && try(startswith(var.deployment_bundle, "gs://"), false) + remote_bucket_name = local.is_remote_bundle ? split("/", var.deployment_bundle)[2] : null + remote_bundle_artifact = local.is_remote_bundle ? join("/", slice(split("/", var.deployment_bundle), 3, length(split("/", var.deployment_bundle)))) : null + should_provision_artifacts_bucket = var.custom_artifacts_bucket_name == null && (!local.is_remote_bundle || var.enable_remote_resources) file_name_with_sha1 = local.is_remote_bundle ? sha1(var.deployment_bundle) : replace(module.psoxy_package.filename, ".jar", "_${filesha1(module.psoxy_package.path_to_deployment_jar)}.zip") @@ -229,7 +230,7 @@ data "archive_file" "source" { # trivy:ignore:AVD-GCP-0078 # trivy:ignore:AVD-GCP-0077 resource "google_storage_bucket" "artifacts" { - count = local.is_remote_bundle ? 0 : 1 + count = local.should_provision_artifacts_bucket ? 1 : 0 project = var.project_id name = coalesce(var.custom_artifacts_bucket_name, "${var.project_id}-${var.environment_id_prefix}artifacts-bucket") @@ -259,7 +260,10 @@ resource "google_storage_bucket_object" "function" { } locals { - artifact_bucket_name = local.is_remote_bundle ? local.remote_bucket_name : google_storage_bucket.artifacts[0].name + # NOTE: not coalesce; Terraform evaluates all coalesce() args even when an earlier one is non-null, + # and coalesce fails when every argument is null (e.g. prebuilt gs:// bundle without remote resources). + artifacts_bucket_name = var.custom_artifacts_bucket_name != null ? var.custom_artifacts_bucket_name : try(google_storage_bucket.artifacts[0].name, null) + deployment_bundle_bucket = local.is_remote_bundle ? local.remote_bucket_name : local.artifacts_bucket_name deployment_bundle_object_name = local.is_remote_bundle ? local.remote_bundle_artifact : google_storage_bucket_object.function[0].name } @@ -535,7 +539,12 @@ output "gcp_project" { } output "artifacts_bucket_name" { - value = local.artifact_bucket_name + value = local.artifacts_bucket_name +} + +output "deployment_bundle_bucket" { + value = local.deployment_bundle_bucket + description = "GCS bucket containing the Cloud Function deployment bundle (may differ from artifacts_bucket_name when using a gs:// deployment_bundle)" } output "artifacts_bucket_id" { diff --git a/infra/modules/gcp/variables.tf b/infra/modules/gcp/variables.tf index e742eb2e7..df1f73f56 100644 --- a/infra/modules/gcp/variables.tf +++ b/infra/modules/gcp/variables.tf @@ -96,10 +96,16 @@ variable "gcp_principals_authorized_to_test" { variable "custom_artifacts_bucket_name" { type = string - description = "name of bucket to use for custom artifacts, if you want something other than default" + description = "Name of an existing GCS bucket to use for deployment artifacts and remote resources (rules, NLP models, etc.). If null, one will be provisioned when needed for a local deployment bundle or when enable_remote_resources is true." default = null } +variable "enable_remote_resources" { + type = bool + description = "Whether to provision an artifacts bucket for remote resources when one is not otherwise needed for deployment (e.g. with a gs:// deployment_bundle)." + default = false +} + variable "support_bulk_mode" { type = bool diff --git a/tools/lib/deployment-bundle.sh b/tools/lib/deployment-bundle.sh index 9276a4168..e69162b99 100644 --- a/tools/lib/deployment-bundle.sh +++ b/tools/lib/deployment-bundle.sh @@ -159,15 +159,60 @@ deployment_bundle_public_path() { esac } +deployment_bundle_s3_parts() { + local bundle_path="$1" + + if [[ ! "$bundle_path" =~ ^s3://([^/]+)/(.+)$ ]]; then + return 1 + fi + + DEPLOYMENT_BUNDLE_S3_BUCKET="${BASH_REMATCH[1]}" + DEPLOYMENT_BUNDLE_S3_KEY="${BASH_REMATCH[2]}" + DEPLOYMENT_BUNDLE_S3_REGION="us-east-1" + if [[ "$DEPLOYMENT_BUNDLE_S3_BUCKET" =~ ^psoxy-public-artifacts-(.+)$ ]]; then + DEPLOYMENT_BUNDLE_S3_REGION="${BASH_REMATCH[1]}" + fi +} + +deployment_bundle_s3_to_http_url() { + local bundle_path="$1" + local bucket key region + + if ! deployment_bundle_s3_parts "$bundle_path"; then + return 1 + fi + bucket="$DEPLOYMENT_BUNDLE_S3_BUCKET" + key="$DEPLOYMENT_BUNDLE_S3_KEY" + region="$DEPLOYMENT_BUNDLE_S3_REGION" + + printf 'https://%s.s3.%s.amazonaws.com/%s' "$bucket" "$region" "$key" +} + deployment_bundle_public_exists() { local bundle_path="$1" case "$bundle_path" in s3://*) - if ! command -v aws >/dev/null 2>&1; then - return 1 + if command -v aws >/dev/null 2>&1 && deployment_bundle_s3_parts "$bundle_path"; then + # head-object only needs object-level access; s3 ls requires s3:ListBucket on the bucket + if aws s3api head-object \ + --bucket "$DEPLOYMENT_BUNDLE_S3_BUCKET" \ + --key "$DEPLOYMENT_BUNDLE_S3_KEY" \ + --region "$DEPLOYMENT_BUNDLE_S3_REGION" \ + >/dev/null 2>&1; then + return 0 + fi + fi + if command -v curl >/dev/null 2>&1; then + local http_url="" http_code="" + if http_url="$(deployment_bundle_s3_to_http_url "$bundle_path")"; then + # follow redirects, then require a final 2xx (3xx alone is not success) + http_code="$(curl -sSIL --max-redirs 5 -o /dev/null -w "%{http_code}" "$http_url" 2>/dev/null)" || return 1 + [[ "$http_code" =~ ^2[0-9]{2}$ ]] + return $? + fi fi - aws s3 ls "$bundle_path" >/dev/null 2>&1 + return 1 ;; gs://*) if command -v gsutil >/dev/null 2>&1; then