From 4b8c8e98e958d1154c2b2a1f77cc5e208c8f7966 Mon Sep 17 00:00:00 2001 From: Alexander Amiri Date: Sun, 8 Mar 2026 23:27:57 +0100 Subject: [PATCH 1/4] Implement Cognito and Identity Center sync in team-provisioner Lambda Wire Cognito internal pool ID and Identity Store ID to the Lambda via env vars. Add IAM permissions for cognito-idp and identitystore APIs. Replace stub functions with full sync: create/update groups, add/remove members by email lookup. --- .../lambda-src/team_provisioner/handler.py | 226 +++++++++++++++--- terraform/platform/lambdas/main.tf | 31 +++ terraform/platform/lambdas/variables.tf | 16 ++ terraform/platform/main.tf | 3 + terraform/platform/variables.tf | 6 + 5 files changed, 254 insertions(+), 28 deletions(-) diff --git a/terraform/lambda-src/team_provisioner/handler.py b/terraform/lambda-src/team_provisioner/handler.py index f2cc0ee..c7f710c 100644 --- a/terraform/lambda-src/team_provisioner/handler.py +++ b/terraform/lambda-src/team_provisioner/handler.py @@ -1,14 +1,14 @@ -"""Team provisioner — syncs teams across Google Workspace, GitHub, and AWS Budgets. +"""Team provisioner — syncs teams across Google, GitHub, AWS Budgets, Cognito, and Identity Center. -Trigger: repository_dispatch from javaBin/registry (via function URL or SNS). +Trigger: Direct Lambda invocation from javaBin/registry CI (via javabin-ci-registry OIDC role). Event payload contains team YAML definitions. Integrations: - Google Admin SDK: Create/sync Google Workspace groups (domain-wide delegation SA) - GitHub API: Create/sync GitHub teams (GitHub App installation token) - AWS Budgets: Create team-level budget scoped to `team` tag -- Cognito: TODO — pools not deployed yet -- IAM Identity Center: TODO — not configured yet +- Cognito: Create/sync groups in internal user pool, assign members by email +- IAM Identity Center: Create/sync groups in identity store, assign members by email """ import base64 @@ -32,6 +32,8 @@ ssm = boto3.client("ssm") budgets_client = boto3.client("budgets") +cognito_client = boto3.client("cognito-idp") +identitystore_client = boto3.client("identitystore") GOOGLE_SA_PARAM = os.environ.get( "GOOGLE_SA_PARAM", "/javabin/platform/google-admin-sa" @@ -51,6 +53,8 @@ ACCOUNT_ID = os.environ.get("ACCOUNT_ID", "") GITHUB_ORG = os.environ.get("GITHUB_ORG", "javaBin") ALERTS_TOPIC_ARN = os.environ.get("ALERTS_TOPIC_ARN", "") +COGNITO_INTERNAL_POOL_ID = os.environ.get("COGNITO_INTERNAL_POOL_ID", "") +IDENTITY_STORE_ID = os.environ.get("IDENTITY_STORE_ID", "") # Cache credentials across invocations within the same Lambda container _credential_cache = {} @@ -512,36 +516,202 @@ def sync_budget(team): # --------------------------------------------------------------------------- def sync_cognito_group(team): - """Sync team to Cognito user pool groups. - - TODO: Implement when Cognito user pools are deployed. - Needs: internal_pool_id, external_pool_id from identity module outputs. - Steps: - 1. Create group in internal pool (for java.no members) - 2. Create group in external pool (if team has external members) - 3. Assign users to groups via cognito-idp:AdminAddUserToGroup + """Create or update a Cognito group in the internal user pool and sync membership. + + Members are matched by email (username in Cognito). Members not yet in the + pool are skipped (they'll be added on next sync after they sign up). """ - logger.info( - "Cognito sync skipped — pools not deployed (team: %s)", team["name"] - ) - return {"skipped": True, "reason": "cognito_pools_not_deployed"} + if not COGNITO_INTERNAL_POOL_ID: + logger.info("Cognito sync skipped — no pool ID configured (team: %s)", team["name"]) + return {"skipped": True, "reason": "cognito_pool_not_configured"} + + team_name = team["name"] + pool_id = COGNITO_INTERNAL_POOL_ID + group_name = f"team-{team_name}" + + # Create or update group + try: + cognito_client.get_group(GroupName=group_name, UserPoolId=pool_id) + cognito_client.update_group( + GroupName=group_name, + UserPoolId=pool_id, + Description=team["description"], + ) + logger.info("Updated Cognito group %s", group_name) + except cognito_client.exceptions.ResourceNotFoundException: + cognito_client.create_group( + GroupName=group_name, + UserPoolId=pool_id, + Description=team["description"], + ) + logger.info("Created Cognito group %s", group_name) + + # Current members in the group + current_users = set() + next_token = None + while True: + kwargs = {"GroupName": group_name, "UserPoolId": pool_id, "Limit": 60} + if next_token: + kwargs["NextToken"] = next_token + resp = cognito_client.list_users_in_group(**kwargs) + for user in resp.get("Users", []): + for attr in user.get("Attributes", []): + if attr["Name"] == "email": + current_users.add(attr["Value"].lower()) + next_token = resp.get("NextToken") + if not next_token: + break + + # Desired members — resolve email to Cognito username + desired_emails = set() + for member in team.get("members", []): + m = _normalize_member(member) + email = m.get("email", "").lower() + if not email: + continue + desired_emails.add(email) + + if email not in current_users: + # Look up user by email in the pool + users_resp = cognito_client.list_users( + UserPoolId=pool_id, + Filter=f'email = "{email}"', + Limit=1, + ) + if users_resp.get("Users"): + username = users_resp["Users"][0]["Username"] + cognito_client.admin_add_user_to_group( + UserPoolId=pool_id, + Username=username, + GroupName=group_name, + ) + logger.info("Added %s to Cognito group %s", email, group_name) + else: + logger.info("User %s not in Cognito pool — skipping", email) + + # Remove members no longer in the team + for email in current_users - desired_emails: + users_resp = cognito_client.list_users( + UserPoolId=pool_id, + Filter=f'email = "{email}"', + Limit=1, + ) + if users_resp.get("Users"): + username = users_resp["Users"][0]["Username"] + cognito_client.admin_remove_user_from_group( + UserPoolId=pool_id, + Username=username, + GroupName=group_name, + ) + logger.info("Removed %s from Cognito group %s", email, group_name) + + return {"synced": True, "group": group_name, "member_count": len(desired_emails)} def sync_identity_center_group(team): - """Sync team to IAM Identity Center groups. - - TODO: Implement when Identity Center is configured. - Needs: identity_store_id from identity module outputs. - Steps: - 1. Create group via identitystore:CreateGroup - 2. Resolve user IDs via identitystore:ListUsers - 3. Assign via identitystore:CreateGroupMembership + """Create or update an Identity Center group and sync membership. + + Members are matched by email in the identity store. Members not yet + provisioned in Identity Center are skipped. """ - logger.info( - "Identity Center sync skipped — not configured (team: %s)", - team["name"], + if not IDENTITY_STORE_ID: + logger.info( + "Identity Center sync skipped — no store ID configured (team: %s)", + team["name"], + ) + return {"skipped": True, "reason": "identity_store_not_configured"} + + team_name = team["name"] + group_name = f"team-{team_name}" + store_id = IDENTITY_STORE_ID + + # Find or create group + group_id = None + groups_resp = identitystore_client.list_groups( + IdentityStoreId=store_id, + Filters=[{"AttributePath": "DisplayName", "AttributeValue": group_name}], ) - return {"skipped": True, "reason": "identity_center_not_configured"} + if groups_resp.get("Groups"): + group_id = groups_resp["Groups"][0]["GroupId"] + logger.info("Found Identity Center group %s (%s)", group_name, group_id) + else: + create_resp = identitystore_client.create_group( + IdentityStoreId=store_id, + DisplayName=group_name, + Description=team["description"], + ) + group_id = create_resp["GroupId"] + logger.info("Created Identity Center group %s (%s)", group_name, group_id) + + # Current group memberships + current_members = {} # user_id -> membership_id + next_token = None + while True: + kwargs = {"IdentityStoreId": store_id, "GroupId": group_id} + if next_token: + kwargs["NextToken"] = next_token + resp = identitystore_client.list_group_memberships(**kwargs) + for membership in resp.get("GroupMemberships", []): + member_id = membership.get("MemberId", {}).get("UserId") + if member_id: + current_members[member_id] = membership["MembershipId"] + next_token = resp.get("NextToken") + if not next_token: + break + + # Build reverse lookup: user_id -> email for current members + current_emails = {} # email -> user_id + for user_id in current_members: + try: + user = identitystore_client.describe_user( + IdentityStoreId=store_id, UserId=user_id + ) + for email_obj in user.get("Emails", []): + current_emails[email_obj["Value"].lower()] = user_id + except Exception: + logger.warning("Could not describe user %s", user_id) + + # Desired members + desired_emails = set() + for member in team.get("members", []): + m = _normalize_member(member) + email = m.get("email", "").lower() + if not email: + continue + desired_emails.add(email) + + if email not in current_emails: + # Look up user by email + users_resp = identitystore_client.list_users( + IdentityStoreId=store_id, + Filters=[{"AttributePath": "UserName", "AttributeValue": email}], + ) + if users_resp.get("Users"): + user_id = users_resp["Users"][0]["UserId"] + try: + identitystore_client.create_group_membership( + IdentityStoreId=store_id, + GroupId=group_id, + MemberId={"UserId": user_id}, + ) + logger.info("Added %s to Identity Center group %s", email, group_name) + except identitystore_client.exceptions.ConflictException: + logger.info("User %s already in Identity Center group %s", email, group_name) + else: + logger.info("User %s not in Identity Center — skipping", email) + + # Remove members no longer in the team + for email, user_id in current_emails.items(): + if email not in desired_emails: + membership_id = current_members.get(user_id) + if membership_id: + identitystore_client.delete_group_membership( + IdentityStoreId=store_id, + MembershipId=membership_id, + ) + logger.info("Removed %s from Identity Center group %s", email, group_name) + + return {"synced": True, "group": group_name, "member_count": len(desired_emails)} # --------------------------------------------------------------------------- diff --git a/terraform/platform/lambdas/main.tf b/terraform/platform/lambdas/main.tf index a35074b..c899d04 100644 --- a/terraform/platform/lambdas/main.tf +++ b/terraform/platform/lambdas/main.tf @@ -401,6 +401,35 @@ resource "aws_iam_role_policy" "team_provisioner" { ] Resource = "arn:aws:budgets::${var.aws_account_id}:budget/javabin-team-*" }, + { + Sid = "CognitoGroupSync" + Effect = "Allow" + Action = [ + "cognito-idp:CreateGroup", + "cognito-idp:GetGroup", + "cognito-idp:UpdateGroup", + "cognito-idp:ListUsersInGroup", + "cognito-idp:AdminAddUserToGroup", + "cognito-idp:AdminRemoveUserFromGroup", + "cognito-idp:ListUsers", + ] + Resource = var.internal_user_pool_arn + }, + { + Sid = "IdentityStoreSync" + Effect = "Allow" + Action = [ + "identitystore:CreateGroup", + "identitystore:DescribeGroup", + "identitystore:ListGroups", + "identitystore:CreateGroupMembership", + "identitystore:ListGroupMemberships", + "identitystore:DeleteGroupMembership", + "identitystore:ListUsers", + ] + # Identity Store API requires * — store ID scoped via env var + Resource = "*" + }, ] }) } @@ -528,6 +557,8 @@ resource "aws_lambda_function" "team_provisioner" { ACCOUNT_ID = var.aws_account_id GITHUB_ORG = "javaBin" ALERTS_TOPIC_ARN = var.alerts_topic_arn + COGNITO_INTERNAL_POOL_ID = var.internal_user_pool_id + IDENTITY_STORE_ID = var.identity_store_id } } } diff --git a/terraform/platform/lambdas/variables.tf b/terraform/platform/lambdas/variables.tf index 1380989..b25e191 100644 --- a/terraform/platform/lambdas/variables.tf +++ b/terraform/platform/lambdas/variables.tf @@ -28,3 +28,19 @@ variable "compliance_reporter_identities" { type = list(string) } +variable "internal_user_pool_id" { + description = "Cognito internal user pool ID" + type = string +} + +variable "internal_user_pool_arn" { + description = "Cognito internal user pool ARN" + type = string +} + +variable "identity_store_id" { + description = "IAM Identity Center identity store ID" + type = string + default = "" +} + diff --git a/terraform/platform/main.tf b/terraform/platform/main.tf index 069ae12..d3150f4 100644 --- a/terraform/platform/main.tf +++ b/terraform/platform/main.tf @@ -59,6 +59,9 @@ module "lambdas" { alerts_topic_arn = module.monitoring.alerts_topic_arn security_topic_arn = module.monitoring.security_topic_arn compliance_reporter_identities = var.auto_tagger_identities + internal_user_pool_id = module.identity.internal_user_pool_id + internal_user_pool_arn = module.identity.internal_user_pool_arn + identity_store_id = var.identity_store_id } module "identity" { diff --git a/terraform/platform/variables.tf b/terraform/platform/variables.tf index 997d649..dd7b658 100644 --- a/terraform/platform/variables.tf +++ b/terraform/platform/variables.tf @@ -67,6 +67,12 @@ variable "certificate_arn" { default = "" } +variable "identity_store_id" { + description = "IAM Identity Center identity store ID (from terraform/org/ outputs)" + type = string + default = "d-9967444724" +} + variable "auto_tagger_identities" { description = "IAM identity substrings allowed to trigger auto-tagging" type = list(string) From 60e5a59242716b55a1867e153cad967bc66e883e Mon Sep 17 00:00:00 2001 From: Alexander Amiri Date: Sun, 8 Mar 2026 23:34:24 +0100 Subject: [PATCH 2/4] Add missing filter block to service-bucket lifecycle config --- terraform/modules/service-bucket/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/modules/service-bucket/main.tf b/terraform/modules/service-bucket/main.tf index d6f9223..b1fa577 100644 --- a/terraform/modules/service-bucket/main.tf +++ b/terraform/modules/service-bucket/main.tf @@ -45,6 +45,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "this" { rule { id = "expire-objects" status = "Enabled" + filter {} expiration { days = var.expire_days From 356586d22edd03dc153d5f5a175fa3a842e74d68 Mon Sep 17 00:00:00 2001 From: Alexander Amiri Date: Sun, 8 Mar 2026 23:38:51 +0100 Subject: [PATCH 3/4] Add ignore_changes on task_definition to ECS service Prevents tf-apply from reverting the image tag set by ecs-deploy. TF manages the service infrastructure, ecs-deploy manages images. --- terraform/modules/ecs-service/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/modules/ecs-service/main.tf b/terraform/modules/ecs-service/main.tf index 88641c3..faefc5a 100644 --- a/terraform/modules/ecs-service/main.tf +++ b/terraform/modules/ecs-service/main.tf @@ -82,5 +82,9 @@ resource "aws_ecs_service" "this" { container_port = var.port } + lifecycle { + ignore_changes = [task_definition] + } + depends_on = [aws_ecs_task_definition.this] } From a5319e3ad0a7d4d302c86aad394f187bef4583b5 Mon Sep 17 00:00:00 2001 From: Alexander Amiri Date: Mon, 9 Mar 2026 23:55:59 +0100 Subject: [PATCH 4/4] Add generic Terraform module expander MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registry-driven preprocessor that reads module source files and generates expanded raw resource definitions from app.yaml. The registry (registry.py) is the single source of truth for module mappings — adding a new resource type means adding a registry entry, nothing else. The expander (expand-modules.py) is module-agnostic: it reads module .tf files from disk, substitutes variables, renames resources, and resolves cross-module references via output maps. Replaces generate-terraform.sh (not yet deleted — CI update pending). --- scripts/expand-modules.py | 923 ++++++++++++++++++++++++++++++++++++++ scripts/registry.py | 410 +++++++++++++++++ 2 files changed, 1333 insertions(+) create mode 100644 scripts/expand-modules.py create mode 100644 scripts/registry.py diff --git a/scripts/expand-modules.py b/scripts/expand-modules.py new file mode 100644 index 0000000..401b54d --- /dev/null +++ b/scripts/expand-modules.py @@ -0,0 +1,923 @@ +#!/usr/bin/env python3 +"""Terraform module expander — generates raw resource definitions from app.yaml. + +Reads the module registry (registry.py) and module source files, then expands +them into concrete .tf files with actual aws_* resource definitions. + +The registry is the ONLY place that knows about modules. This script is +module-agnostic — it reads module .tf files from disk and performs: + 1. Variable substitution (var.xxx -> concrete values) + 2. Resource renaming (.this -> .routing, .bucket_uploads, etc.) + 3. Internal reference rewriting (resource_type.this -> resource_type.renamed) + 4. Cross-module reference resolution (via output_map) + 5. Construct resolution (count, dynamic, for_each) + +Usage: + python3 expand-modules.py + +Required env vars: APP_SERVICE, AWS_ACCOUNT_ID, AWS_REGION, TF_ROOT +Optional env vars: PLATFORM_ROOT (defaults to .platform) +""" + +import os +import re +import subprocess +import sys +import yaml + +# Import registry from the same directory +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from registry import ( + REGISTRY, PROJECT, DOMAIN, + BACKEND_TEMPLATE, PROVIDERS_TEMPLATE, OUTPUTS_TEMPLATE, +) + +GENERATED_MARKER = "# GENERATED FROM app.yaml — do not edit, changes will be overwritten" + + +# --------------------------------------------------------------------------- +# YAML helpers +# --------------------------------------------------------------------------- + +def load_yaml(path): + with open(path) as f: + return yaml.safe_load(f) + + +def yaml_get(data, dot_path, default=None): + """Traverse a nested dict by dot-separated path.""" + keys = dot_path.split(".") + current = data + for key in keys: + if not isinstance(current, dict) or key not in current: + return default + current = current[key] + return current + + +# --------------------------------------------------------------------------- +# Expression resolver (the mini DSL) +# --------------------------------------------------------------------------- + +def resolve_expr(expr, yaml_data, env_vars, item=None, ref_resolver=None): + """Resolve a registry DSL expression to a concrete value. + + Returns (value, is_reference) where is_reference=True means the value + is a Terraform expression (not a quoted string). + """ + if expr.startswith("yaml:"): + return _resolve_yaml(expr[5:], yaml_data) + if expr.startswith("const:"): + val = expr[6:] + if val in ("true", "false"): + return val, True # bare boolean + if val == "null": + return "null", True + try: + float(val) + return val, True # bare number + except ValueError: + return val, False # string + if expr.startswith("env:"): + return env_vars.get(expr[4:], ""), False + if expr.startswith("item:"): + return _resolve_item(expr[5:], item) + if expr.startswith("ref:"): + ref = expr[4:] + if ref_resolver: + resolved = ref_resolver(ref) + return resolved, True # always a TF reference + return f"UNRESOLVED:{ref}", True + if expr.startswith("list:"): + inner = expr[5:] + # Check if it's a raw TF reference (contains dots, not a DSL expression) + if not any(inner.startswith(p) for p in ("yaml:", "const:", "env:", "item:", "ref:", "expr:")): + # Bare TF reference like "data.aws_sns_topic.alerts.arn" + return f"[{inner}]", True + inner_val, is_ref = resolve_expr(inner, yaml_data, env_vars, item, ref_resolver) + if is_ref: + return f"[{inner_val}]", True + return f'["{inner_val}"]', True + if expr.startswith("expr:"): + return _resolve_interpolation(expr[5:], yaml_data, env_vars, item, ref_resolver) + if expr.startswith("collect:"): + # Handled specially by the main loop + return expr, True + + return expr, False + + +def _resolve_yaml(path_with_default, yaml_data): + parts = path_with_default.split("|") + path = parts[0] + default = None + for part in parts[1:]: + if part.startswith("default:"): + default = part[8:] + + val = yaml_get(yaml_data, path) + if val is None: + if default is not None: + if default in ("true", "false"): + return default, True + if default == "null": + return "null", True + try: + float(default) + return default, True + except ValueError: + return default, False + raise ValueError(f"Required YAML path '{path}' not found and no default") + + if isinstance(val, bool): + return str(val).lower(), True + if isinstance(val, (int, float)): + return str(val), True + return str(val), False + + +def _resolve_item(path_with_default, item): + parts = path_with_default.split("|") + field = parts[0] + default = None + for part in parts[1:]: + if part.startswith("default:"): + default = part[8:] + + val = item.get(field) if item else None + if val is None: + if default is not None: + if default in ("true", "false"): + return default, True + if default == "null": + return "null", True + if default == "": + return "", False + try: + float(default) + return default, True + except ValueError: + return default, False + raise ValueError(f"Required item field '{field}' not found") + + if isinstance(val, bool): + return str(val).lower(), True + if isinstance(val, (int, float)): + return str(val), True + return str(val), False + + +def _resolve_interpolation(template, yaml_data, env_vars, item, ref_resolver): + """Resolve ${ref:xxx} and ${yaml:xxx} within an interpolation string. + + Example: "${ref:ecr.repository_url}:latest" + → resolves ref → "aws_ecr_repository.ecr.repository_url" + → produces: "${aws_ecr_repository.ecr.repository_url}:latest" + """ + parts = [] + has_refs = False + last_end = 0 + + for match in re.finditer(r'\$\{([^}]+)\}', template): + parts.append(template[last_end:match.start()]) + inner = match.group(1) + val, is_ref = resolve_expr(inner, yaml_data, env_vars, item, ref_resolver) + if is_ref: + parts.append("${" + val + "}") + has_refs = True + else: + parts.append(val) + last_end = match.end() + + parts.append(template[last_end:]) + resolved = "".join(parts) + + if has_refs: + return f'"{resolved}"', True + return resolved, False + + +# --------------------------------------------------------------------------- +# Module source reader +# --------------------------------------------------------------------------- + +def read_module_source(platform_root, module_path): + """Read the main.tf content from a module directory.""" + main_tf = os.path.join(platform_root, module_path, "main.tf") + if not os.path.exists(main_tf): + raise FileNotFoundError(f"Module source not found: {main_tf}") + with open(main_tf) as f: + return f.read() + + +def read_module_variables(platform_root, module_path): + """Read variable names, types, and defaults from variables.tf.""" + var_tf = os.path.join(platform_root, module_path, "variables.tf") + if not os.path.exists(var_tf): + return {} + with open(var_tf) as f: + content = f.read() + + variables = {} + for match in re.finditer( + r'variable\s+"(\w+)"\s*\{(.*?)\n\}', + content, re.DOTALL + ): + name = match.group(1) + body = match.group(2) + var_type = "string" + type_match = re.search(r'type\s*=\s*(\S+)', body) + if type_match: + var_type = type_match.group(1) + + default = None + default_match = re.search(r'default\s*=\s*(.+)', body) + if default_match: + raw = default_match.group(1).strip() + # Clean up trailing comments or braces + raw = re.sub(r'\s*#.*$', '', raw) + if raw.endswith('"'): + # String default: extract between quotes + str_match = re.search(r'"(.*)"', raw) + if str_match: + default = str_match.group(1) + elif raw in ("true", "false"): + default = raw + elif raw == "null": + default = None + else: + try: + default = str(int(raw)) + except ValueError: + try: + default = str(float(raw)) + except ValueError: + default = raw + + variables[name] = {"type": var_type, "default": default} + return variables + + +# --------------------------------------------------------------------------- +# HCL expansion engine +# --------------------------------------------------------------------------- + +def expand_hcl(source, var_values, rename_from, rename_to, extra_renames=None): + """Expand a module's main.tf by substituting variables and renaming resources. + + Args: + source: The raw HCL text from the module's main.tf + var_values: Dict of {var_name: (value, is_reference)} + rename_from: Original resource name suffix (usually "this") + rename_to: New resource name suffix (e.g., "routing", "bucket_uploads") + extra_renames: Additional name mappings for non-"this" resources (e.g., "dlq" -> "queue_x_dlq") + """ + result = source + + # 1. Substitute var.xxx references + for var_name, (value, is_ref) in var_values.items(): + # First handle "${var.xxx}" interpolation patterns (replace with just the value) + result = re.sub( + rf'"\$\{{var\.{re.escape(var_name)}\}}"', + lambda m, v=value, r=is_ref: v if r else f'"{v}"', + result, + ) + # Then handle ${var.xxx} inside larger strings (inline substitution) + result = re.sub( + rf'\$\{{var\.{re.escape(var_name)}\}}', + lambda m, v=value: str(v), + result, + ) + # Finally handle bare var.xxx references + if is_ref: + result = re.sub( + rf'\bvar\.{re.escape(var_name)}\b', + lambda m, v=value: v, + result, + ) + else: + result = re.sub( + rf'\bvar\.{re.escape(var_name)}\b', + lambda m, v=value: f'"{v}"', + result, + ) + + # 2. Rename resources and internal references + # Find all resource/data types declared in the source + declared = set() + for match in re.finditer(r'(?:resource|data)\s+"(\w+)"\s+"(\w+)"', result): + declared.add((match.group(1), match.group(2))) + + renames = extra_renames or {} + renames[rename_from] = rename_to + + for res_type, old_name in declared: + new_name = renames.get(old_name, f"{rename_to}_{old_name}" if old_name != rename_from else rename_to) + # Rename in resource/data declarations + result = re.sub( + rf'((?:resource|data)\s+"{re.escape(res_type)}"\s+)"{re.escape(old_name)}"', + rf'\1"{new_name}"', + result, + ) + # Rename in references (e.g., aws_s3_bucket.this.arn -> aws_s3_bucket.bucket_uploads.arn) + result = re.sub( + rf'\b{re.escape(res_type)}\.{re.escape(old_name)}\b', + f'{res_type}.{new_name}', + result, + ) + + return result + + +def strip_comments_header(hcl_text): + """Remove leading comment blocks (################################ style).""" + lines = hcl_text.split("\n") + result = [] + skipping_header = True + for line in lines: + if skipping_header: + if line.strip().startswith("#") or line.strip() == "": + continue + skipping_header = False + result.append(line) + return "\n".join(result) + + +def remove_for_each_resource(hcl_text, resource_pattern): + """Remove a resource block that uses for_each (will be expanded separately).""" + # Match the resource block and remove it + pattern = rf'resource\s+"{resource_pattern.split(".")[0]}"\s+"{resource_pattern.split(".")[1]}"\s*\{{[^}}]*for_each[^}}]*\}}' + # Use a more robust block matcher + lines = hcl_text.split("\n") + result = [] + skip_depth = 0 + skipping = False + + resource_type = resource_pattern.split(".")[0] + resource_name = resource_pattern.split(".")[1] + target = f'resource "{resource_type}" "{resource_name}"' + + for line in lines: + if not skipping and target in line: + skipping = True + skip_depth = 0 + if skipping: + skip_depth += line.count("{") - line.count("}") + if skip_depth <= 0 and "{" in line or (skip_depth <= 0 and "}" in line): + if skip_depth <= 0: + skipping = False + continue + continue + result.append(line) + + return "\n".join(result) + + +def resolve_count_blocks(hcl_text, variable, include): + """Remove count lines and optionally remove entire resource blocks.""" + if include: + # Keep the resource but remove the count line + return re.sub(r'\s*count\s*=\s*[^\n]+\n', '\n', hcl_text) + else: + # Remove entire resource blocks that have this count + # This is complex — for now we just remove count and let TF handle it + # Actually, if not included, we should remove the whole resource + return hcl_text # TODO: implement full block removal + + +def resolve_dynamic_block(hcl_text, block_name, include): + """Resolve a dynamic block to either its content or nothing.""" + if include: + # Replace dynamic "name" { for_each = ... content { ... } } + # with just the content block + pattern = rf'(\s*)dynamic\s+"{re.escape(block_name)}"\s*\{{[^}}]*content\s*\{{(.*?)\}}\s*\}}' + match = re.search(pattern, hcl_text, re.DOTALL) + if match: + indent = match.group(1) + content = match.group(2) + # Replace dynamic.value references with the actual variable ref + replacement = f'{indent}{block_name} {{{content}}}' + hcl_text = hcl_text[:match.start()] + replacement + hcl_text[match.end():] + else: + # Remove the entire dynamic block + pattern = rf'\s*dynamic\s+"{re.escape(block_name)}"\s*\{{.*?\}}\s*\}}' + hcl_text = re.sub(pattern, '', hcl_text, flags=re.DOTALL) + + return hcl_text + + +# --------------------------------------------------------------------------- +# Collect expressions (cross-wiring between modules) +# --------------------------------------------------------------------------- + +def collect_access_policies(instances): + """Build the additional_policy_jsons map from all collection instances.""" + policies = {} + for module_id, inst_name, entry in instances: + exports = entry.get("exports", {}) + if exports.get("access_policy_json"): + output_map = entry.get("output_map", {}) + ref_template = output_map.get("access_policy_json", "") + ref = ref_template.replace("{instance}", f"{entry['rename']}_{inst_name}") + policies[f"{entry['rename']}-{inst_name}"] = ref + return policies + + +def collect_env_vars(yaml_data, instances): + """Build the environment map from YAML + auto-wired resources.""" + env_map = {} + # Static env vars from YAML + yaml_env = yaml_get(yaml_data, "environment") or {} + for k, v in yaml_env.items(): + env_map[k] = (str(v), False) # (value, is_reference) + + # Auto-wired from collection modules + for module_id, inst_name, entry in instances: + exports = entry.get("exports", {}) + env_export = exports.get("env_var") + if env_export and env_export.get("target") == "environment": + # Check if the YAML item has the env field set + yaml_list_path = entry.get("yaml_list", "") + items = yaml_get(yaml_data, yaml_list_path) or [] + for item in items: + if item.get(entry.get("instance_key")) == inst_name: + env_field = env_export.get("yaml_field", "env") + env_var_name = item.get(env_field) + if env_var_name: + output = env_export["output"] + output_map = entry.get("output_map", {}) + ref_template = output_map.get(output, "") + ref = ref_template.replace("{instance}", f"{entry['rename']}_{inst_name}") + env_map[env_var_name] = (ref, True) + return env_map + + +def collect_secret_vars(yaml_data, instances): + """Build the secrets map from auto-wired secret resources.""" + secret_map = {} + for module_id, inst_name, entry in instances: + exports = entry.get("exports", {}) + env_export = exports.get("env_var") + if env_export and env_export.get("target") == "secrets": + yaml_list_path = entry.get("yaml_list", "") + items = yaml_get(yaml_data, yaml_list_path) or [] + for item in items: + if item.get(entry.get("instance_key")) == inst_name: + env_field = env_export.get("yaml_field", "env") + env_var_name = item.get(env_field) + if env_var_name: + output = env_export["output"] + output_map = entry.get("output_map", {}) + ref_template = output_map.get(output, "") + ref = ref_template.replace("{instance}", f"{entry['rename']}_{inst_name}") + secret_map[env_var_name] = (ref, True) + return secret_map + + +def format_map_literal(entries): + """Format a dict as an HCL map literal.""" + if not entries: + return "{}" + lines = [] + for k, (v, is_ref) in entries.items(): + if is_ref: + lines.append(f' "{k}" = {v}') + else: + lines.append(f' "{k}" = "{v}"') + return "{\n" + "\n".join(lines) + "\n }" + + +# --------------------------------------------------------------------------- +# Reference resolver +# --------------------------------------------------------------------------- + +def build_ref_resolver(registry_entries): + """Build a function that resolves ref:module.output expressions.""" + output_maps = {} + for entry in registry_entries: + output_maps[entry["id"]] = entry.get("output_map", {}) + + def resolver(ref): + parts = ref.split(".", 1) + if len(parts) != 2: + return f"UNRESOLVED:{ref}" + module_id, output_name = parts + omap = output_maps.get(module_id, {}) + if output_name in omap: + return omap[output_name] + return f"UNRESOLVED:{ref}" + + return resolver + + +# --------------------------------------------------------------------------- +# File writer +# --------------------------------------------------------------------------- + +def should_write(filepath): + """Check if a file can be overwritten (has GENERATED marker or doesn't exist).""" + if not os.path.exists(filepath): + return True + with open(filepath) as f: + first_line = f.readline() + return GENERATED_MARKER in first_line + + +def write_file(filepath, content): + """Write a file with the GENERATED marker.""" + if not should_write(filepath): + print(f" skipped {os.path.basename(filepath)} (user-managed)") + return + with open(filepath, "w") as f: + f.write(content) + print(f" wrote {os.path.basename(filepath)}") + + +# --------------------------------------------------------------------------- +# Main orchestrator +# --------------------------------------------------------------------------- + +def main(): + app_yaml_path = "app.yaml" + if not os.path.exists(app_yaml_path): + print("No app.yaml found, skipping generation") + return + + # Environment + service = os.environ.get("APP_SERVICE", "") + account_id = os.environ.get("AWS_ACCOUNT_ID", "") + region = os.environ.get("AWS_REGION", "eu-central-1") + tf_root = os.environ.get("TF_ROOT", "terraform") + platform_root = os.environ.get("PLATFORM_ROOT", ".platform") + + yaml_data = load_yaml(app_yaml_path) + app_name = yaml_data.get("name", "") + app_team = yaml_data.get("team", "unknown") + app_host = yaml_get(yaml_data, "routing.host") or yaml_data.get("domain", "") + + if not app_name: + print("ERROR: app.yaml must have a 'name' field") + sys.exit(1) + + print(f"Expanding modules for {app_name}") + os.makedirs(tf_root, exist_ok=True) + + env_vars = { + "AWS_ACCOUNT_ID": account_id, + "AWS_REGION": region, + "APP_SERVICE": service, + } + + ref_resolver = build_ref_resolver(REGISTRY) + + # -- Pass 1: Discover all collection instances -- + collection_instances = [] # (module_id, instance_name, entry) + for entry in REGISTRY: + if entry["cardinality"] != "collection": + continue + yaml_list_path = entry.get("yaml_list", "") + items = yaml_get(yaml_data, yaml_list_path) or [] + instance_key = entry.get("instance_key", "name") + for item in items: + inst_name = item.get(instance_key, "") + if inst_name: + collection_instances.append((entry["id"], inst_name, entry)) + + # -- Pre-compute collect expressions -- + access_policies = collect_access_policies(collection_instances) + env_var_map = collect_env_vars(yaml_data, collection_instances) + secret_var_map = collect_secret_vars(yaml_data, collection_instances) + + # -- Pass 2: Expand each module -- + file_contents = {} # output_file -> content + + for entry in REGISTRY: + module_id = entry["id"] + output_file = entry["output_file"] + + # Check condition + condition = entry.get("condition") + if condition: + try: + val, _ = resolve_expr(condition, yaml_data, env_vars) + if val in ("false", "0", "", "null"): + continue + except ValueError: + continue + + # Read module source + try: + source = read_module_source(platform_root, entry["module_path"]) + except FileNotFoundError as e: + print(f" WARNING: {e}") + continue + + # Read module variable defaults for unbound vars + mod_vars = read_module_variables(platform_root, entry["module_path"]) + + if entry["cardinality"] == "singleton": + hcl = _expand_singleton( + entry, source, yaml_data, env_vars, ref_resolver, + access_policies, env_var_map, secret_var_map, + mod_vars, + ) + content = file_contents.get(output_file, "") + file_contents[output_file] = content + hcl + + elif entry["cardinality"] == "collection": + yaml_list_path = entry.get("yaml_list", "") + items = yaml_get(yaml_data, yaml_list_path) or [] + if not items: + continue + + instance_key = entry.get("instance_key", "name") + all_hcl = "" + + for item in items: + inst_name = item.get(instance_key, "") + if not inst_name: + continue + hcl = _expand_collection_item( + entry, source, yaml_data, env_vars, ref_resolver, item, inst_name, + mod_vars, + ) + all_hcl += hcl + + if all_hcl: + content = file_contents.get(output_file, "") + file_contents[output_file] = content + all_hcl + + # -- Write boilerplate files -- + write_file( + os.path.join(tf_root, "backend.tf"), + BACKEND_TEMPLATE.format( + project=PROJECT, account_id=account_id, + service=service, region=region, + ), + ) + write_file( + os.path.join(tf_root, "providers.tf"), + PROVIDERS_TEMPLATE.format( + region=region, project=PROJECT, + service=service, team=app_team, + ), + ) + write_file( + os.path.join(tf_root, "outputs.tf"), + OUTPUTS_TEMPLATE.format(host=app_host), + ) + + # -- Write expanded module files -- + for filename, content in file_contents.items(): + filepath = os.path.join(tf_root, filename) + write_file(filepath, GENERATED_MARKER + "\n" + content) + + # -- Write extra data sources -- + for entry in REGISTRY: + extra = entry.get("extra_data_sources", []) + if extra and entry["output_file"] in file_contents: + filepath = os.path.join(tf_root, entry["output_file"]) + with open(filepath, "a") as f: + for ds in extra: + f.write(f'\ndata "{ds["type"]}" "{ds["name"]}" {{\n') + f.write(f' {ds["body"]}\n') + f.write("}\n") + + # -- Fingerprint -- + import hashlib + with open(app_yaml_path, "rb") as f: + sha = hashlib.sha256(f.read()).hexdigest() + with open(os.path.join(tf_root, ".app-yaml-hash"), "w") as f: + f.write(sha + "\n") + + # -- Format -- + subprocess.run(["terraform", "fmt", tf_root], capture_output=True) + print("Expansion complete") + + +def _expand_singleton(entry, source, yaml_data, env_vars, ref_resolver, + access_policies, env_var_map, secret_var_map, + mod_vars=None): + """Expand a singleton module entry into HCL.""" + rename_to = entry.get("rename", entry["id"]) + + # Resolve variables + var_values = {} + for var_name, expr in entry.get("vars", {}).items(): + if expr == "collect:access_policy_json": + var_values[var_name] = (format_map_literal( + {k: (v, True) for k, v in access_policies.items()} + ), True) + elif expr == "collect:env_vars": + var_values[var_name] = (format_map_literal(env_var_map), True) + elif expr == "collect:secret_vars": + var_values[var_name] = (format_map_literal(secret_var_map), True) + else: + var_values[var_name] = resolve_expr( + expr, yaml_data, env_vars, ref_resolver=ref_resolver, + ) + + # Apply module variable defaults for any vars not in registry + if mod_vars: + for vname, vinfo in mod_vars.items(): + if vname not in var_values and vinfo.get("default") is not None: + default = vinfo["default"] + is_ref = default in ("true", "false") or default.isdigit() + var_values[vname] = (default, is_ref) + + extra_renames = {} + + # Handle constructs + hcl = source + constructs = entry.get("constructs", {}) + + # Remove for_each resources (expanded separately) + fe = constructs.get("for_each_expand") + if fe: + resource_pattern = fe["resource"] + # Remove the for_each resource block from source + hcl = _remove_resource_block(hcl, resource_pattern) + + # Resolve count-based conditionals + cr = constructs.get("count_resolve") + if cr: + var = cr["variable"] + val, _ = var_values.get(var, ("true", True)) + include = val not in ("false", "0", "null") + for res in cr.get("resources", []): + if not include: + hcl = _remove_resource_block(hcl, res) + else: + hcl = re.sub(r'\s*count\s*=\s*[^\n]+\n', '\n', hcl, count=1) + + # Expand the HCL + hcl = expand_hcl(hcl, var_values, "this", rename_to, extra_renames) + + # Strip module-level comments + hcl = _clean_expanded(hcl) + + # Append expanded for_each resources + if fe: + variable = fe["variable"] + policies_val = var_values.get(variable, ({}, True))[0] + if isinstance(policies_val, str) and policies_val != "{}": + hcl += _expand_for_each_policies( + access_policies, rename_to, + ) + + return hcl + + +def _expand_collection_item(entry, source, yaml_data, env_vars, ref_resolver, item, inst_name, + mod_vars=None): + """Expand a collection module entry for one item.""" + rename_prefix = entry.get("rename", entry["id"]) + rename_to = f"{rename_prefix}_{inst_name}" + + # Resolve variables + var_values = {} + for var_name, expr in entry.get("vars", {}).items(): + var_values[var_name] = resolve_expr( + expr, yaml_data, env_vars, item=item, ref_resolver=ref_resolver, + ) + + # Apply module variable defaults for any vars not in registry + if mod_vars: + for vname, vinfo in mod_vars.items(): + if vname not in var_values and vinfo.get("default") is not None: + default = vinfo["default"] + is_ref = default in ("true", "false") or default.isdigit() + var_values[vname] = (default, is_ref) + + hcl = source + + # Handle constructs + constructs = entry.get("constructs", {}) + + # Resolve count-based conditionals + cr = constructs.get("count_resolve") + if cr: + var = cr["variable"] + val, _ = var_values.get(var, ("0", True)) + condition = cr.get("condition", "truthy") + if condition == "greater_than_zero": + include = val not in ("0", "0.0", "false", "null") + else: + include = val not in ("false", "0", "null") + for res in cr.get("resources", []): + if not include: + hcl = _remove_resource_block(hcl, res) + else: + hcl = re.sub(r'\s*count\s*=\s*[^\n]+\n', '\n', hcl, count=1) + + # Resolve dynamic blocks + dr = constructs.get("dynamic_resolve") + if dr: + var = dr["variable"] + val, _ = var_values.get(var, ("null", True)) + include = val not in ("null", "", "false") + block_name = dr["block_name"] + hcl = resolve_dynamic_block(hcl, block_name, include) + if include: + # Replace attribute.value with the actual value + hcl = re.sub(r'\battribute\.value\b', val, hcl) + + # Handle extra renames for non-"this" resources (e.g., SQS "dlq") + extra_renames = {} + # Find all resource names in source that aren't "this" + for match in re.finditer(r'resource\s+"\w+"\s+"(\w+)"', source): + name = match.group(1) + if name != "this": + extra_renames[name] = f"{rename_to}_{name}" + + # Also handle data sources + for match in re.finditer(r'data\s+"\w+"\s+"(\w+)"', source): + name = match.group(1) + extra_renames[name] = f"{rename_to}_{name}" + + hcl = expand_hcl(hcl, var_values, "this", rename_to, extra_renames) + hcl = _clean_expanded(hcl) + + return hcl + + +def _remove_resource_block(hcl, resource_pattern): + """Remove a resource block identified by type.name pattern.""" + parts = resource_pattern.split(".") + if len(parts) != 2: + return hcl + res_type, res_name = parts + + lines = hcl.split("\n") + result = [] + skip_depth = 0 + skipping = False + + for line in lines: + stripped = line.strip() + if not skipping: + if (f'resource "{res_type}" "{res_name}"' in stripped or + f'data "{res_type}" "{res_name}"' in stripped): + skipping = True + skip_depth = 0 + for ch in line: + if ch == '{': + skip_depth += 1 + elif ch == '}': + skip_depth -= 1 + if skip_depth <= 0 and '{' in line: + skipping = skip_depth > 0 + continue + result.append(line) + else: + for ch in line: + if ch == '{': + skip_depth += 1 + elif ch == '}': + skip_depth -= 1 + if skip_depth <= 0: + skipping = False + # Don't append — we're removing this block + + return "\n".join(result) + + +def _expand_for_each_policies(access_policies, role_rename): + """Generate individual aws_iam_role_policy resources for each policy.""" + blocks = [] + for policy_name, policy_ref in access_policies.items(): + safe_name = policy_name.replace("-", "_") + blocks.append(f""" +resource "aws_iam_role_policy" "policy_{safe_name}" {{ + name = "{policy_name}" + role = aws_iam_role.{role_rename}.id + policy = {policy_ref} +}} +""") + return "\n".join(blocks) + + +def _clean_expanded(hcl): + """Clean up expanded HCL: remove excessive blank lines, leading/trailing whitespace.""" + # Remove lines that are just "########..." comments + lines = hcl.split("\n") + result = [] + for line in lines: + if re.match(r'^\s*#{10,}\s*$', line): + continue + if re.match(r'^\s*#\s*\w', line): + # Keep meaningful comments + result.append(line) + elif line.strip().startswith("#"): + continue # Skip other comment lines + else: + result.append(line) + + # Collapse multiple blank lines + text = "\n".join(result) + text = re.sub(r'\n{3,}', '\n\n', text) + return text.strip() + "\n" + + +if __name__ == "__main__": + main() diff --git a/scripts/registry.py b/scripts/registry.py new file mode 100644 index 0000000..5b01788 --- /dev/null +++ b/scripts/registry.py @@ -0,0 +1,410 @@ +"""Module registry — maps app.yaml sections to platform Terraform modules. + +This is the ONLY place modules are defined. Adding a new resource type to the +platform means adding a registry entry here. The expander (expand-modules.py) +reads this registry and the module source files to generate expanded Terraform. +""" + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +PROJECT = "javabin" +DOMAIN = "javazone.no" +MODULE_ROOT = "terraform/modules" + +# --------------------------------------------------------------------------- +# Registry entries +# +# Each entry describes one module and how it maps to app.yaml: +# +# id Unique key, used for resource renaming and cross-references +# module_path Path to module source (relative to platform repo root) +# output_file Generated .tf file name in the app repo +# cardinality "singleton" (one per app) or "collection" (N from YAML list) +# condition If set, a DSL expression that must be truthy to emit this module +# vars Dict of module_variable -> value expression (DSL) +# rename Resource name to replace "this" with (singletons) +# output_map Maps module output names -> expanded resource attribute paths +# constructs Hints for complex HCL constructs (for_each, dynamic, count) +# yaml_list (collections) Dot-path to the YAML array +# instance_key (collections) YAML field used as instance name +# exports (collections) What this module contributes to cross-wiring +# --------------------------------------------------------------------------- + +REGISTRY = [ + # ------------------------------------------------------------------ + # platform-data: shared infrastructure data sources (always emitted) + # ------------------------------------------------------------------ + { + "id": "platform", + "module_path": f"{MODULE_ROOT}/platform-data", + "output_file": "platform.tf", + "cardinality": "singleton", + "vars": { + "project": f"const:{PROJECT}", + "domain": f"const:{DOMAIN}", + }, + "rename": "platform", + "output_map": { + "vpc_id": "data.aws_vpc.platform_main.id", + "private_subnet_ids": "data.aws_subnets.platform_private.ids", + "public_subnet_ids": "data.aws_subnets.platform_public.ids", + "ecs_tasks_security_group_id": "data.aws_security_group.platform_ecs_tasks.id", + "alb_arn": "data.aws_lb.platform_main.arn", + "alb_dns_name": "data.aws_lb.platform_main.dns_name", + "alb_zone_id": "data.aws_lb.platform_main.zone_id", + "alb_arn_suffix": "data.aws_lb.platform_main.arn_suffix", + "https_listener_arn": "data.aws_lb_listener.platform_https.arn", + "ecs_cluster_id": "data.aws_ecs_cluster.platform_main.id", + "ecs_cluster_name": "data.aws_ecs_cluster.platform_main.cluster_name", + "execution_role_arn": "data.aws_iam_role.platform_ecs_execution.arn", + "route53_zone_id": "data.aws_route53_zone.platform_main.zone_id", + "developer_boundary_arn": "data.aws_iam_policy.platform_developer_boundary.arn", + }, + }, + + # ------------------------------------------------------------------ + # ecr-repo: container registry + # ------------------------------------------------------------------ + { + "id": "ecr", + "module_path": f"{MODULE_ROOT}/ecr-repo", + "output_file": "ecr.tf", + "cardinality": "singleton", + "vars": { + "name": "yaml:name", + }, + "rename": "ecr", + "output_map": { + "repository_url": "aws_ecr_repository.ecr.repository_url", + "repository_arn": "aws_ecr_repository.ecr.arn", + "repository_name": "aws_ecr_repository.ecr.name", + }, + }, + + # ------------------------------------------------------------------ + # service-routing: ALB target group + listener rule + DNS + # ------------------------------------------------------------------ + { + "id": "routing", + "module_path": f"{MODULE_ROOT}/service-routing", + "output_file": "routing.tf", + "cardinality": "singleton", + "vars": { + "name": "yaml:name", + "project": f"const:{PROJECT}", + "vpc_id": "ref:platform.vpc_id", + "port": "yaml:compute.port|default:8000", + "health_check_path": "yaml:compute.health_check|default:/health", + "health_check_matcher": "yaml:compute.health_check_matcher|default:200", + "https_listener_arn": "ref:platform.https_listener_arn", + "listener_rule_priority": "yaml:routing.priority", + "host_header": "yaml:routing.host", + "route53_zone_id": "ref:platform.route53_zone_id", + "alb_dns_name": "ref:platform.alb_dns_name", + "alb_zone_id": "ref:platform.alb_zone_id", + }, + "rename": "routing", + "output_map": { + "target_group_arn": "aws_lb_target_group.routing.arn", + "target_group_arn_suffix": "aws_lb_target_group.routing.arn_suffix", + "dns_name": "aws_route53_record.routing.fqdn", + }, + }, + + # ------------------------------------------------------------------ + # service-role: ECS task IAM role + policies + # ------------------------------------------------------------------ + { + "id": "task_role", + "module_path": f"{MODULE_ROOT}/service-role", + "output_file": "iam.tf", + "cardinality": "singleton", + "vars": { + "name": "yaml:name", + "project": f"const:{PROJECT}", + "region": "env:AWS_REGION", + "aws_account_id": "env:AWS_ACCOUNT_ID", + "permissions_boundary_arn": "ref:platform.developer_boundary_arn", + "additional_policy_jsons": "collect:access_policy_json", + }, + "rename": "task", + "output_map": { + "role_arn": "aws_iam_role.task.arn", + "role_id": "aws_iam_role.task.id", + "role_name": "aws_iam_role.task.name", + }, + "constructs": { + # for_each on additional_policy_jsons: expand into N named resources + "for_each_expand": { + "resource": "aws_iam_role_policy.additional", + "variable": "additional_policy_jsons", + }, + }, + }, + + # ------------------------------------------------------------------ + # ecs-service: log group + task definition + ECS service + # ------------------------------------------------------------------ + { + "id": "service", + "module_path": f"{MODULE_ROOT}/ecs-service", + "output_file": "service.tf", + "cardinality": "singleton", + "vars": { + "name": "yaml:name", + "cluster_id": "ref:platform.ecs_cluster_id", + "image": "expr:${ref:ecr.repository_url}:latest", + "cpu": "yaml:compute.cpu|default:512", + "memory": "yaml:compute.memory|default:1024", + "port": "yaml:compute.port|default:8000", + "desired_count": "yaml:compute.desired_count|default:1", + "execution_role_arn": "ref:platform.execution_role_arn", + "task_role_arn": "ref:task_role.role_arn", + "subnet_ids": "ref:platform.private_subnet_ids", + "security_group_ids": "list:ref:platform.ecs_tasks_security_group_id", + "target_group_arn": "ref:routing.target_group_arn", + "region": "env:AWS_REGION", + "environment": "collect:env_vars", + "secrets": "collect:secret_vars", + }, + "rename": "main", + "output_map": { + "service_name": "aws_ecs_service.main.name", + "task_definition_arn": "aws_ecs_task_definition.main.arn", + "log_group_name": "aws_cloudwatch_log_group.main.name", + }, + }, + + # ------------------------------------------------------------------ + # service-alarm: CloudWatch alarms (conditional) + # ------------------------------------------------------------------ + { + "id": "alarms", + "module_path": f"{MODULE_ROOT}/service-alarm", + "output_file": "alarms.tf", + "cardinality": "singleton", + "condition": "yaml:alarms.enabled|default:true", + "vars": { + "name": "yaml:name", + "project": f"const:{PROJECT}", + "service": "yaml:name", + "cluster_name": "ref:platform.ecs_cluster_name", + "sns_topic_arns": "list:data.aws_sns_topic.alerts.arn", + "cpu_threshold": "yaml:alarms.cpu_threshold|default:80", + "memory_threshold": "yaml:alarms.memory_threshold|default:80", + "error_5xx_threshold": "yaml:alarms.error_5xx_threshold|default:10", + "enable_alb_alarms": "const:true", + "target_group_arn_suffix": "ref:routing.target_group_arn_suffix", + "alb_arn_suffix": "ref:platform.alb_arn_suffix", + }, + "rename": "alarm", + "extra_data_sources": [ + { + "type": "aws_sns_topic", + "name": "alerts", + "body": f'name = "{PROJECT}-alerts"', + }, + ], + "constructs": { + # count-based conditional resources: resolve at gen time + "count_resolve": { + "variable": "enable_alb_alarms", + "resources": ["aws_cloudwatch_metric_alarm.unhealthy_targets", + "aws_cloudwatch_metric_alarm.target_5xx"], + }, + }, + }, + + # ------------------------------------------------------------------ + # Collection modules: one set of resources per YAML list item + # ------------------------------------------------------------------ + { + "id": "bucket", + "module_path": f"{MODULE_ROOT}/service-bucket", + "output_file": "buckets.tf", + "cardinality": "collection", + "yaml_list": "resources.buckets", + "instance_key": "name", + "vars": { + "name": "item:name", + "project": "yaml:name", + "aws_account_id": "env:AWS_ACCOUNT_ID", + "service": "yaml:name", + "versioning": "item:versioning|default:true", + "expire_days": "item:expire_days|default:0", + }, + "rename": "bucket", + "output_map": { + "bucket_name": "aws_s3_bucket.{instance}.id", + "bucket_arn": "aws_s3_bucket.{instance}.arn", + "access_policy_json": "data.aws_iam_policy_document.{instance}_access.json", + }, + "exports": { + "access_policy_json": True, + "env_var": {"output": "bucket_name", "yaml_field": "env", "target": "environment"}, + }, + "constructs": { + "count_resolve": { + "variable": "expire_days", + "condition": "greater_than_zero", + "resources": ["aws_s3_bucket_lifecycle_configuration.this"], + }, + }, + }, + { + "id": "database", + "module_path": f"{MODULE_ROOT}/service-database", + "output_file": "databases.tf", + "cardinality": "collection", + "yaml_list": "resources.databases", + "instance_key": "name", + "vars": { + "name": "item:name", + "project": "yaml:name", + "service": "yaml:name", + "hash_key": "item:hash_key|default:id", + "hash_key_type": "item:hash_key_type|default:S", + "range_key": "item:range_key|default:null", + "range_key_type": "item:range_key_type|default:S", + "ttl_attribute": "item:ttl_attribute|default:", + }, + "rename": "db", + "output_map": { + "table_name": "aws_dynamodb_table.{instance}.name", + "table_arn": "aws_dynamodb_table.{instance}.arn", + "access_policy_json": "data.aws_iam_policy_document.{instance}_access.json", + }, + "exports": { + "access_policy_json": True, + "env_var": {"output": "table_name", "yaml_field": "env", "target": "environment"}, + }, + "constructs": { + "dynamic_resolve": { + "variable": "range_key", + "block_name": "attribute", + "condition": "not_null", + }, + }, + }, + { + "id": "secret", + "module_path": f"{MODULE_ROOT}/service-secret", + "output_file": "secrets.tf", + "cardinality": "collection", + "yaml_list": "resources.secrets", + "instance_key": "name", + "vars": { + "name": "item:name", + "project": "yaml:name", + "service": "yaml:name", + "description": "item:description|default:", + }, + "rename": "secret", + "output_map": { + "secret_arn": "aws_secretsmanager_secret.{instance}.arn", + "access_policy_json": "data.aws_iam_policy_document.{instance}_access.json", + }, + "exports": { + "access_policy_json": True, + "env_var": {"output": "secret_arn", "yaml_field": "env", "target": "secrets"}, + }, + }, + { + "id": "queue", + "module_path": f"{MODULE_ROOT}/service-queue", + "output_file": "queues.tf", + "cardinality": "collection", + "yaml_list": "resources.queues", + "instance_key": "name", + "vars": { + "name": "item:name", + "project": "yaml:name", + "service": "yaml:name", + "visibility_timeout_seconds": "item:visibility_timeout|default:30", + "retention_seconds": "item:retention_seconds|default:345600", + "max_receive_count": "item:max_receive_count|default:3", + }, + "rename": "queue", + "output_map": { + "queue_url": "aws_sqs_queue.{instance}.url", + "queue_arn": "aws_sqs_queue.{instance}.arn", + "dlq_url": "aws_sqs_queue.{instance}_dlq.url", + "access_policy_json": "data.aws_iam_policy_document.{instance}_access.json", + }, + "exports": { + "access_policy_json": True, + "env_var": {"output": "queue_url", "yaml_field": "env", "target": "environment"}, + }, + }, +] + + +# --------------------------------------------------------------------------- +# Boilerplate templates (not module-derived) +# --------------------------------------------------------------------------- + +BACKEND_TEMPLATE = """\ +# GENERATED FROM app.yaml — do not edit, changes will be overwritten +terraform {{ + backend "s3" {{ + bucket = "{project}-terraform-state-{account_id}" + key = "apps/{service}/terraform.tfstate" + region = "{region}" + dynamodb_table = "{project}-terraform-app-locks" + encrypt = true + }} +}} +""" + +PROVIDERS_TEMPLATE = """\ +# GENERATED FROM app.yaml — do not edit, changes will be overwritten +terraform {{ + required_version = ">= 1.5" + + required_providers {{ + aws = {{ + source = "hashicorp/aws" + version = "~> 5.0" + }} + }} +}} + +provider "aws" {{ + region = "{region}" + + default_tags {{ + tags = {{ + project = "{project}" + managed-by = "terraform" + service = "{service}" + team = "{team}" + environment = "production" + }} + }} +}} +""" + +OUTPUTS_TEMPLATE = """\ +# GENERATED FROM app.yaml — do not edit, changes will be overwritten +output "service_url" {{ + value = "https://{host}" +}} + +output "ecr_url" {{ + value = aws_ecr_repository.ecr.repository_url +}} + +output "service_name" {{ + value = aws_ecs_service.main.name +}} + +output "task_role_arn" {{ + value = aws_iam_role.task.arn +}} + +output "log_group" {{ + value = aws_cloudwatch_log_group.main.name +}} +"""