fix remote resource bug

CHANGELOG.md

@@ -5,6 +5,9 @@ in each release's notes.
 
 Changes to be including in future/planned release notes will be added here.
 
+## [Unreleased]
+- `aws`/`gcp`: fix Terraform plan failure when `enable_remote_resources = true` but no artifacts bucket exists (e.g. with a prebuilt `deployment_bundle`). When remote resources are enabled, an artifacts bucket is now provisioned if one is not already created or provided via `artifacts_bucket_name` / `custom_artifacts_bucket_name`.
+
 ## [0.6.5](https://github.com/Worklytics/psoxy/releases/tag/v0.6.5)
 - added `claude-enterprise-analytics` connector in **beta**; imports per-user daily activity, token usage, and cost data from the [Claude Enterprise Analytics API](https://support.claude.com/en/articles/13703965-claude-enterprise-analytics-api-reference-guide); see [docs/sources/anthropic/claude-enterprise-analytics/README.md](docs/sources/anthropic/claude-enterprise-analytics/README.md)
 

docs/configuration/remote-resources.md

@@ -29,12 +29,17 @@ mounted locally.
 
 ## Terraform Configuration
 
-By default, the host modules in this repository (`aws-host` and `gcp-host`) will configure the
-`REMOTE_RESOURCE_BUCKET` for you if you set the `enable_remote_resources` variable to `true`. This
-automatically wires the **artifacts bucket** (used for deployment bundles) as the remote resource bucket.
+Remote resources are **opt-in**. Set `enable_remote_resources = true` on the host module
+(`aws-host` or `gcp-host`) when you want psoxy to load rules, NLP models, or other assets from
+the artifacts bucket at runtime. The host module does not infer this from your connector list.
+
+When enabled, the host module uses the artifacts bucket — either one you provide
+(`artifacts_bucket_name` / `custom_artifacts_bucket_name`), one already provisioned for a local
+deployment bundle, or a newly provisioned bucket when using a prebuilt `s3://` / `gs://`
+`deployment_bundle`.
 
 > [!IMPORTANT]
-> If you configure an existing bucket (e.g., by providing `artifacts_bucket_name`), the bucket must already exist.
+> If you supply an existing bucket (`artifacts_bucket_name` / `custom_artifacts_bucket_name`), it must already exist.
 >
 > The Terraform runner (the credentials running the `terraform` command) must have sufficient IAM permissions on that bucket to apply permissions, since it will grant read access to the proxy's service account or Lambda execution role.
 
@@ -46,7 +51,6 @@ module "psoxy" {
 
   # ... existing configuration ...
 
-  # Enable remote resource loading from the artifacts S3 bucket
   enable_remote_resources = true
 }
 ```
@@ -63,7 +67,6 @@ module "psoxy" {
 
   # ... existing configuration ...
 
-  # Enable remote resource loading from the artifacts GCS bucket
   enable_remote_resources = true
 }
 ```

infra/modules/aws-host/main.tf

@@ -110,6 +110,7 @@ module "psoxy" {
   enable_webhook_testing             = local.enable_webhook_testing
   webhook_allow_origins              = distinct(flatten([for v in var.webhook_collectors : v.allow_origins]))
   artifacts_bucket_name              = var.artifacts_bucket_name
+  enable_remote_resources            = var.enable_remote_resources
   allowed_data_access_ip_blocks      = var.allowed_data_access_ip_blocks
   allowed_webhook_ip_blocks          = var.allowed_webhook_ip_blocks
 }
@@ -127,6 +128,7 @@ locals {
   connector_instance_resource_path = { for k, v in merge(var.api_connectors, var.bulk_connectors, var.webhook_collectors) :
     k => "${local.shared_resource_path}${replace(upper(k), "-", "_")}/"
   }
+  remote_resources_enabled = var.enable_remote_resources && module.psoxy.artifacts_bucket_name != null
 
   # convert custom_side_outputs to the format expected by the psoxy module
   custom_original_side_outputs = { for k, v in var.custom_side_outputs :
@@ -261,9 +263,9 @@ module "api_connector" {
     var.general_environment_variables,
   )
 
-  remote_resource_bucket        = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null
-  remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null
-  remote_resource_shared_path   = var.enable_remote_resources ? local.shared_resource_path : null
+  remote_resource_bucket        = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null
+  remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null
+  remote_resource_shared_path   = local.remote_resources_enabled ? local.shared_resource_path : null
 }
 
 
@@ -349,9 +351,9 @@ module "bulk_connector" {
     var.general_environment_variables
   )
 
-  remote_resource_bucket        = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null
-  remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null
-  remote_resource_shared_path   = var.enable_remote_resources ? local.shared_resource_path : null
+  remote_resource_bucket        = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null
+  remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null
+  remote_resource_shared_path   = local.remote_resources_enabled ? local.shared_resource_path : null
 }
 
 
@@ -401,9 +403,9 @@ module "webhook_collectors" {
     var.general_environment_variables,
   )
 
-  remote_resource_bucket        = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null
-  remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null
-  remote_resource_shared_path   = var.enable_remote_resources ? local.shared_resource_path : null
+  remote_resource_bucket        = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null
+  remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null
+  remote_resource_shared_path   = local.remote_resources_enabled ? local.shared_resource_path : null
 }
 
 # Policy to allow test caller to invoke webhook collector urls and sign webhook requests

infra/modules/aws-host/variables.tf

@@ -468,13 +468,12 @@ variable "todo_step" {
 
 variable "artifacts_bucket_name" {
   type        = string
-  description = "Name of an existing S3 bucket to use for deployment artifacts. If null, one will be provisioned if needed."
+  description = "Name of an existing S3 bucket to use for deployment artifacts and remote resources (rules, NLP models, etc.). If null, one will be provisioned when needed for a local deployment bundle or when enable_remote_resources is true."
   default     = null
 }
 
-
 variable "enable_remote_resources" {
   type        = bool
-  description = "**beta** Whether to enable remote resource loading from the artifacts S3 bucket (rules, NLP models, etc.). When true, sets REMOTE_RESOURCE_BUCKET env var and grants s3:GetObject to each Lambda. Default will change to `true` in next major version."
-  default     = false # will change to true in 0.7.x
+  description = "**beta** Whether to enable remote resource loading from the artifacts S3 bucket (rules, NLP models, etc.). When true, sets REMOTE_RESOURCE_BUCKET env var and grants s3:GetObject to each Lambda. Provisions an artifacts bucket if one is not already created or provided."
+  default     = false
 }

infra/modules/aws-proxy-lambda/main.tf

@@ -98,8 +98,8 @@ resource "aws_lambda_function" "instance" {
       length(var.path_to_shared_ssm_parameters) > 0 ? { PATH_TO_SHARED_CONFIG = var.path_to_shared_ssm_parameters } : {},
       local.is_instance_ssm_prefix_default ? {} : { PATH_TO_INSTANCE_CONFIG = var.path_to_instance_ssm_parameters },
       var.remote_resource_bucket != null ? { REMOTE_RESOURCE_BUCKET = var.remote_resource_bucket } : {},
-      var.remote_resource_instance_path != null ? { INSTANCE_RESOURCE_PATH = var.remote_resource_instance_path } : {},
-      var.remote_resource_shared_path != null ? { SHARED_RESOURCE_PATH = var.remote_resource_shared_path } : {},
+      var.remote_resource_bucket != null && var.remote_resource_instance_path != null ? { INSTANCE_RESOURCE_PATH = var.remote_resource_instance_path } : {},
+      var.remote_resource_bucket != null && var.remote_resource_shared_path != null ? { SHARED_RESOURCE_PATH = var.remote_resource_shared_path } : {},
     )
   }
 
@@ -301,14 +301,14 @@ locals {
   remote_resource_instance_prefix = var.remote_resource_instance_path != null ? trimsuffix(var.remote_resource_instance_path, "/") : ""
   remote_resource_shared_prefix   = var.remote_resource_shared_path != null ? trimsuffix(var.remote_resource_shared_path, "/") : ""
 
-  remote_resource_s3_object_arns = distinct(compact([
+  remote_resource_s3_object_arns = var.remote_resource_bucket != null ? distinct(compact([
     var.remote_resource_instance_path != null ? (
       local.remote_resource_instance_prefix != "" ? "arn:aws:s3:::${var.remote_resource_bucket}/${local.remote_resource_instance_prefix}/*" : "arn:aws:s3:::${var.remote_resource_bucket}/*"
     ) : "",
     var.remote_resource_shared_path != null ? (
       local.remote_resource_shared_prefix != "" ? "arn:aws:s3:::${var.remote_resource_bucket}/${local.remote_resource_shared_prefix}/*" : "arn:aws:s3:::${var.remote_resource_bucket}/*"
     ) : "",
-  ]))
+  ])) : []
 
   remote_resource_bucket_statements = var.remote_resource_bucket != null ? [{
     Sid = "ReadRemoteResourceBucket"

infra/modules/aws-proxy-lambda/remote_resource_iam.tftest.hcl

@@ -93,3 +93,18 @@ run "duplicate_prefixes_dedupe_to_one_arn" {
     condition     = toset(one([for s in jsondecode(aws_iam_policy.required_resource_access.policy).Statement : s.Resource if s.Sid == "ReadRemoteResourceBucket"])) == toset(["arn:aws:s3:::my-artifacts/shared/*"])
   }
 }
+
+run "paths_without_bucket_skip_iam" {
+  command = plan
+
+  variables {
+    remote_resource_bucket        = null
+    remote_resource_instance_path = "instances/foo/"
+    remote_resource_shared_path   = "shared/models/"
+  }
+
+  assert {
+    error_message = "paths without a bucket should not create ReadRemoteResourceBucket IAM statement"
+    condition     = length([for s in jsondecode(aws_iam_policy.required_resource_access.policy).Statement : s if s.Sid == "ReadRemoteResourceBucket"]) == 0
+  }
+}

infra/modules/aws-webhook-collector/variables.tf

@@ -201,11 +201,9 @@ variable "provision_auth_key" {
 
   validation {
     condition = (
-      var.provision_auth_key == null ||
-      (
-        try(var.provision_auth_key.rotation_days, null) == null ||
-        try(var.provision_auth_key.rotation_days, 0) > 0
-      )
+      var.provision_auth_key == null ? true :
+      var.provision_auth_key.rotation_days == null ? true :
+      var.provision_auth_key.rotation_days > 0
     )
     error_message = "If `provision_auth_key` is provided, `rotation_days` must be a positive number or null."
   }

infra/modules/aws/main.tf

@@ -158,8 +158,8 @@ module "psoxy_package" {
 locals {
   # determine if the JAR is local and should be uploaded directly from plan-time variables
   is_local_jar            = var.deployment_bundle == null || !startswith(coalesce(var.deployment_bundle, "unknown"), "s3://")
-  should_provision_bucket = local.is_local_jar && var.artifacts_bucket_name == null
-  target_artifacts_bucket = var.artifacts_bucket_name != null ? var.artifacts_bucket_name : (local.should_provision_bucket ? aws_s3_bucket.artifacts[0].bucket : null)
+  should_provision_bucket = var.artifacts_bucket_name == null && (local.is_local_jar || var.enable_remote_resources)
+  target_artifacts_bucket = coalesce(var.artifacts_bucket_name, try(aws_s3_bucket.artifacts[0].bucket, null))
   should_upload_object    = local.is_local_jar && (var.artifacts_bucket_name != null || local.should_provision_bucket)
 }
 

infra/modules/aws/variables.tf

@@ -148,10 +148,16 @@ variable "enable_webhook_testing" {
 
 variable "artifacts_bucket_name" {
   type        = string
-  description = "Name of an existing S3 bucket to use for deployment artifacts. If null, one will be provisioned if needed."
+  description = "Name of an existing S3 bucket to use for deployment artifacts and remote resources (rules, NLP models, etc.). If null, one will be provisioned when needed for a local deployment bundle or when enable_remote_resources is true."
   default     = null
 }
 
+variable "enable_remote_resources" {
+  type        = bool
+  description = "Whether to provision an artifacts bucket for remote resources when one is not otherwise needed for deployment (e.g. with an s3:// deployment_bundle)."
+  default     = false
+}
+
 variable "allowed_data_access_ip_blocks" {
   description = <<-EOT
     IPs or CIDR blocks allowed to make data access requests. When non-empty, adds infrastructure-level aws:SourceIp conditions on api-caller role assume-role policies (see docs/configuration/ip-allowlisting.md). Application-layer enforcement is configured separately on proxy Lambdas via the host module.

infra/modules/gcp-host/main.tf

@@ -26,6 +26,7 @@ locals {
   connector_instance_resource_path = { for k, v in merge(var.api_connectors, var.bulk_connectors, var.webhook_collectors) :
     k => "${local.shared_resource_path}${replace(upper(k), "-", "_")}/"
   }
+  remote_resources_enabled = var.enable_remote_resources && module.psoxy.artifacts_bucket_name != null
 
   # rules_file paths may be absolute, relative to the Terraform root module (deployment dir), or
   # relative to psoxy_base_dir (paths into the psoxy repo, eg docs/sources/...)
@@ -99,6 +100,7 @@ module "psoxy" {
   provision_testing_infra           = var.provision_testing_infra
   gcp_principals_authorized_to_test = var.gcp_principals_authorized_to_test
   custom_artifacts_bucket_name      = var.custom_artifacts_bucket_name
+  enable_remote_resources           = var.enable_remote_resources
   support_bulk_mode                 = length(var.bulk_connectors) > 0
   support_webhook_collectors        = length(var.webhook_collectors) > 0
   vpc_config                        = var.vpc_config
@@ -237,7 +239,7 @@ module "api_connector" {
   environment_id_prefix                 = local.environment_id_prefix
   instance_id                           = each.key
   service_account_email                 = google_service_account.api_connectors[each.key].email
-  artifacts_bucket_name                 = module.psoxy.artifacts_bucket_name
+  artifacts_bucket_name                 = module.psoxy.deployment_bundle_bucket
   deployment_bundle_object_name         = module.psoxy.deployment_bundle_object_name
   artifact_repository_id                = module.psoxy.artifact_repository
   vpc_config                            = module.psoxy.vpc_config
@@ -282,9 +284,9 @@ module "api_connector" {
     var.general_environment_variables,
   )
 
-  remote_resource_bucket        = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null
-  remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null
-  remote_resource_shared_path   = var.enable_remote_resources ? local.shared_resource_path : null
+  remote_resource_bucket        = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null
+  remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null
+  remote_resource_shared_path   = local.remote_resources_enabled ? local.shared_resource_path : null
 
   secret_bindings = merge(
     local.secrets_bound_as_env_vars[each.key],
@@ -341,7 +343,7 @@ module "webhook_collector" {
     service_account_id = google_service_account.webhook_collector[each.key].id
     email              = google_service_account.webhook_collector[each.key].email
   }
-  artifacts_bucket_name              = module.psoxy.artifacts_bucket_name
+  artifacts_bucket_name              = module.psoxy.deployment_bundle_bucket
   deployment_bundle_object_name      = module.psoxy.deployment_bundle_object_name
   artifact_repository_id             = module.psoxy.artifact_repository
   path_to_repo_root                  = var.psoxy_base_dir
@@ -378,9 +380,9 @@ module "webhook_collector" {
     var.general_environment_variables,
   )
 
-  remote_resource_bucket        = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null
-  remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null
-  remote_resource_shared_path   = var.enable_remote_resources ? local.shared_resource_path : null
+  remote_resource_bucket        = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null
+  remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null
+  remote_resource_shared_path   = local.remote_resources_enabled ? local.shared_resource_path : null
 
   secret_bindings = module.psoxy.secrets
 
@@ -404,7 +406,7 @@ module "bulk_connector" {
   worklytics_sa_emails              = var.worklytics_sa_emails
   config_parameter_prefix           = local.config_parameter_prefix
   source_kind                       = each.value.source_kind
-  artifacts_bucket_name             = module.psoxy.artifacts_bucket_name
+  artifacts_bucket_name             = module.psoxy.deployment_bundle_bucket
   artifact_repository_id            = module.psoxy.artifact_repository
   deployment_bundle_object_name     = module.psoxy.deployment_bundle_object_name
   psoxy_base_dir                    = var.psoxy_base_dir
@@ -446,9 +448,9 @@ module "bulk_connector" {
     var.general_environment_variables,
   )
 
-  remote_resource_bucket        = var.enable_remote_resources ? module.psoxy.artifacts_bucket_name : null
-  remote_resource_instance_path = var.enable_remote_resources ? local.connector_instance_resource_path[each.key] : null
-  remote_resource_shared_path   = var.enable_remote_resources ? local.shared_resource_path : null
+  remote_resource_bucket        = local.remote_resources_enabled ? module.psoxy.artifacts_bucket_name : null
+  remote_resource_instance_path = local.remote_resources_enabled ? local.connector_instance_resource_path[each.key] : null
+  remote_resource_shared_path   = local.remote_resources_enabled ? local.shared_resource_path : null
 
   depends_on = [
     module.psoxy # some of the set-up IAM grants done there, but not EXPLICITLY passed out as outputs and into above as inputs, are required; so make this explicit

infra/modules/gcp-host/variables.tf

@@ -181,7 +181,7 @@ variable "kms_key_ring" {
 
 variable "custom_artifacts_bucket_name" {
   type        = string
-  description = "name of bucket to use for custom artifacts, if you want something other than default"
+  description = "Name of an existing GCS bucket to use for deployment artifacts and remote resources (rules, NLP models, etc.). If null, one will be provisioned when needed for a local deployment bundle or when enable_remote_resources is true."
   default     = null
 }
 
@@ -463,6 +463,6 @@ variable "max_instances_per_api_connector" {
 
 variable "enable_remote_resources" {
   type        = bool
-  description = "**beta** Whether to enable remote resource loading from the artifacts GCS bucket (rules, NLP models, etc.). When true, sets REMOTE_RESOURCE_BUCKET env var and grants roles/storage.objectViewer to each Cloud Function. Default will change to `true` in next major version."
-  default     = false # will change to true in 0.7.x
+  description = "**beta** Whether to enable remote resource loading from the artifacts GCS bucket (rules, NLP models, etc.). When true, sets REMOTE_RESOURCE_BUCKET env var and grants roles/storage.objectViewer to each Cloud Function. Provisions an artifacts bucket if one is not already created or provided."
+  default     = false
 }