From 6ca2cfc0f29792b3e9f8971b79951dc26ad15c2b Mon Sep 17 00:00:00 2001 From: Alexander Amiri Date: Mon, 16 Mar 2026 21:37:42 +0100 Subject: [PATCH 1/2] Auto-sync repos from GitHub teams, auto-generate TF boilerplate, drop CDK Registration: - Add sync-registered-repos.py: queries GitHub teams for repos, writes registered-apps.auto.tfvars (replaces manual list) - Platform CI runs sync before every plan + supports manual dispatch - Add organization:read permission for GitHub team API access App repo boilerplate: - Add ensure-tf-boilerplate.sh: auto-generates backend.tf (S3 state) and providers.tf (with default_tags for project/team/managed-by) when repos don't have app.yaml - Resolves team from GitHub API at CI runtime - Won't overwrite existing files CDK: - Remove CDK detection from detect.yml (not supported in pipeline) --- .github/workflows/detect.yml | 5 -- .github/workflows/platform-ci.yml | 11 ++- .github/workflows/tf-plan.yml | 7 ++ scripts/ensure-tf-boilerplate.sh | 69 +++++++++++++++++ scripts/sync-registered-repos.py | 125 ++++++++++++++++++++++++++++++ 5 files changed, 211 insertions(+), 6 deletions(-) create mode 100755 scripts/ensure-tf-boilerplate.sh create mode 100755 scripts/sync-registered-repos.py diff --git a/.github/workflows/detect.yml b/.github/workflows/detect.yml index b2725f5..3aaddd3 100644 --- a/.github/workflows/detect.yml +++ b/.github/workflows/detect.yml @@ -21,9 +21,6 @@ on: has_eb: description: ".elasticbeanstalk/ directory exists" value: ${{ jobs.detect.outputs.has_eb }} - has_cdk: - description: "cdk.json exists" - value: ${{ jobs.detect.outputs.has_cdk }} app_name: description: "App name from app.yaml (empty if no app.yaml)" value: ${{ jobs.detect.outputs.app_name }} @@ -39,7 +36,6 @@ jobs: has_maven: ${{ steps.check.outputs.has_maven }} has_pnpm: ${{ steps.check.outputs.has_pnpm }} has_eb: ${{ steps.check.outputs.has_eb }} - has_cdk: ${{ steps.check.outputs.has_cdk }} app_name: ${{ steps.check.outputs.app_name }} steps: - uses: actions/checkout@v6 @@ -53,7 +49,6 @@ jobs: echo "has_maven=$(test -f pom.xml && echo true || echo false)" >> "$GITHUB_OUTPUT" echo "has_pnpm=$(test -f pnpm-lock.yaml && echo true || echo false)" >> "$GITHUB_OUTPUT" echo "has_eb=$(test -d .elasticbeanstalk && echo true || echo false)" >> "$GITHUB_OUTPUT" - echo "has_cdk=$(test -f cdk.json && echo true || echo false)" >> "$GITHUB_OUTPUT" if [ -f app.yaml ]; then echo "app_name=$(grep -m1 '^name:' app.yaml | awk '{print $2}' | tr -d '\"'"'")" >> "$GITHUB_OUTPUT" else diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index 60d177d..0b3b440 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -10,13 +10,16 @@ on: - '.github/workflows/**' pull_request: schedule: - # Drift detection — Monday 06:00 UTC + # Drift detection — Monday 06:00 UTC (runs drift job only) - cron: '0 6 * * 1' + workflow_dispatch: + # Manual trigger — useful for syncing new repos immediately permissions: id-token: write contents: read pull-requests: write + organization: read # Required for sync-registered-repos.py to list GitHub teams concurrency: group: platform-ci-${{ github.ref }} @@ -71,6 +74,12 @@ jobs: aws-region: ${{ env.AWS_REGION }} role-session-name: javabin-platform-plan-${{ github.run_id }} + - name: Sync registered repos from GitHub teams + if: steps.changes.outputs.has_infra_changes == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/sync-registered-repos.py "${{ env.TF_ROOT }}/registered-apps.auto.tfvars" + - name: Terraform Init if: steps.changes.outputs.has_infra_changes == 'true' working-directory: ${{ env.TF_ROOT }} diff --git a/.github/workflows/tf-plan.yml b/.github/workflows/tf-plan.yml index 7eb0ccd..631ce52 100644 --- a/.github/workflows/tf-plan.yml +++ b/.github/workflows/tf-plan.yml @@ -77,6 +77,13 @@ jobs: path: .platform sparse-checkout: scripts + - name: Ensure Terraform boilerplate + env: + AWS_ACCOUNT_ID: ${{ inputs.aws_account_id }} + AWS_REGION: ${{ inputs.aws_region || 'eu-central-1' }} + GH_TOKEN: ${{ github.token }} + run: sh .platform/scripts/ensure-tf-boilerplate.sh "${{ inputs.tf_root }}" + - name: Terraform Init working-directory: ${{ inputs.tf_root }} run: terraform init -input=false diff --git a/scripts/ensure-tf-boilerplate.sh b/scripts/ensure-tf-boilerplate.sh new file mode 100755 index 0000000..8a4ac64 --- /dev/null +++ b/scripts/ensure-tf-boilerplate.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# Auto-generate backend.tf and providers.tf for repos without app.yaml. +# +# If these files already exist (committed by the developer or generated by +# expand-modules.py), they are NOT overwritten. This is a safety net for +# repos that bring their own Terraform without using app.yaml. +# +# Usage: ensure-tf-boilerplate.sh +# +# Env: AWS_ACCOUNT_ID, AWS_REGION (default eu-central-1) +# Uses: GITHUB_REPOSITORY (org/repo), gh CLI for team resolution +# +# Generated files include default_tags with project, team, and managed-by +# so that ALL Terraform-created resources are automatically tagged — +# even if the developer doesn't add tags to their own resources. + +set -e + +TF_ROOT="${1:?Usage: ensure-tf-boilerplate.sh }" +ACCOUNT_ID="${AWS_ACCOUNT_ID:?AWS_ACCOUNT_ID required}" +REGION="${AWS_REGION:-eu-central-1}" +REPO_NAME="${GITHUB_REPOSITORY##*/}" +PROJECT="javabin" + +# Resolve team from GitHub API (first non-platform team the repo belongs to) +TEAM="" +if command -v gh >/dev/null 2>&1; then + TEAM=$(gh api "/repos/${GITHUB_REPOSITORY}/teams" \ + --jq '[.[] | select(.slug != "platform-owners")] | .[0].slug // ""' 2>/dev/null || true) +fi +[ -z "$TEAM" ] && TEAM="unknown" + +# --- Generate backend.tf if missing --- +if [ ! -f "$TF_ROOT/backend.tf" ]; then + cat > "$TF_ROOT/backend.tf" < "$TF_ROOT/providers.tf" < + +Requires: GITHUB_TOKEN env var (with org:read scope) or GitHub App credentials +via GH_APP_ID + GH_APP_KEY env vars (read from SSM if not set). +""" + +import json +import logging +import os +import sys +import urllib.request + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, format="%(message)s") + +GITHUB_ORG = os.environ.get("GITHUB_ORG", "javaBin") +GITHUB_API = "https://api.github.com" + +# Repos that are platform infrastructure, not apps +EXCLUDED_REPOS = {"platform", "registry", ".github"} + + +def github_get(path, token): + """Paginated GitHub API GET.""" + results = [] + url = f"{GITHUB_API}{path}" + while url: + req = urllib.request.Request(url) + req.add_header("Authorization", f"token {token}") + req.add_header("Accept", "application/vnd.github+json") + with urllib.request.urlopen(req) as resp: + results.extend(json.loads(resp.read())) + # Follow pagination + link = resp.headers.get("Link", "") + url = None + for part in link.split(","): + if 'rel="next"' in part: + url = part.split("<")[1].split(">")[0] + return results + + +def get_team_repos(token): + """Get all repos across all org teams, excluding platform repos.""" + teams = github_get(f"/orgs/{GITHUB_ORG}/teams", token) + logger.info("Found %d teams in %s", len(teams), GITHUB_ORG) + + repo_set = set() + for team in teams: + slug = team["slug"] + repos = github_get(f"/orgs/{GITHUB_ORG}/teams/{slug}/repos", token) + team_repo_names = {r["name"] for r in repos} - EXCLUDED_REPOS + if team_repo_names: + logger.info(" %s: %s", slug, ", ".join(sorted(team_repo_names))) + repo_set.update(team_repo_names) + + return sorted(repo_set) + + +def write_tfvars(repos, output_path): + """Write the registered-apps.auto.tfvars file.""" + lines = [ + "# GENERATED by sync-registered-repos.py — do not edit manually", + "# Source of truth: GitHub team repo memberships in javaBin org", + f"# Last synced: {__import__('datetime').datetime.now(__import__('datetime').timezone.utc).isoformat()}", + "", + "registered_app_repos = [", + ] + for repo in repos: + lines.append(f' "{repo}",') + lines.append("]") + lines.append("") + + content = "\n".join(lines) + + # Only write if changed + existing = "" + if os.path.exists(output_path): + with open(output_path) as f: + existing = f.read() + + # Compare ignoring the timestamp line + def strip_timestamp(s): + return "\n".join(l for l in s.splitlines() if not l.startswith("# Last synced:")) + + if strip_timestamp(content) == strip_timestamp(existing): + logger.info("No changes to registered repos") + return False + + with open(output_path, "w") as f: + f.write(content) + logger.info("Wrote %d repos to %s", len(repos), output_path) + return True + + +def main(): + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + output_path = sys.argv[1] + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if not token: + print("GITHUB_TOKEN or GH_TOKEN required", file=sys.stderr) + sys.exit(1) + + repos = get_team_repos(token) + logger.info("Total registered repos: %d", len(repos)) + + changed = write_tfvars(repos, output_path) + if changed: + logger.info("registered-apps.auto.tfvars updated") + sys.exit(0) + + +if __name__ == "__main__": + main() From c22349406239b5810693357c10f84c395432bab2 Mon Sep 17 00:00:00 2001 From: Alexander Amiri Date: Mon, 16 Mar 2026 21:42:32 +0100 Subject: [PATCH 2/2] Fix workflow syntax: remove invalid organization permission organization is not a valid GITHUB_TOKEN permission scope. Also scope repo sync to main branch only (not needed on PRs). --- .github/workflows/platform-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index 0b3b440..637e11a 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -19,7 +19,6 @@ permissions: id-token: write contents: read pull-requests: write - organization: read # Required for sync-registered-repos.py to list GitHub teams concurrency: group: platform-ci-${{ github.ref }} @@ -75,10 +74,10 @@ jobs: role-session-name: javabin-platform-plan-${{ github.run_id }} - name: Sync registered repos from GitHub teams - if: steps.changes.outputs.has_infra_changes == 'true' + if: steps.changes.outputs.has_infra_changes == 'true' && github.ref == 'refs/heads/main' env: GH_TOKEN: ${{ github.token }} - run: python3 scripts/sync-registered-repos.py "${{ env.TF_ROOT }}/registered-apps.auto.tfvars" + run: python3 scripts/sync-registered-repos.py "${{ env.TF_ROOT }}/registered-apps.auto.tfvars" || echo "Sync skipped — will use existing tfvars" - name: Terraform Init if: steps.changes.outputs.has_infra_changes == 'true'