From 1055e2a2e68669423a28866bb1915ac1c48db95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Mon, 22 Jun 2026 06:12:30 +1200 Subject: [PATCH 1/3] ci(aws): mirror published AMIs to satellite regions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AMIs are region-scoped: the AMI registered by register-ami in ap-southeast-2 is only directly launchable from ap-southeast-2. Consumers in other regions would otherwise have to copy-image themselves before they could launch. Add a copy-amis matrix job that fans out one (arch × suite × region) job per target — ap-southeast-6, eu-central-5, ap-south-1, and us-east-1 — and a copy-ami-to-region.sh script that does the copy, waits for the new AMI to reach 'available', tags the backing snapshot, and makes both the copy and its snapshot public. Idempotent: skips if the named AMI already exists in the target region. --- .github/workflows/build.yml | 35 +++++++ scripts/copy-ami-to-region.sh | 182 ++++++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100755 scripts/copy-ami-to-region.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8827591..c45edad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -777,3 +777,38 @@ jobs: # every time the bucket is recreated. run: scripts/register-ami-for-release.sh "${{ matrix.arch }}" "${{ matrix.suite }}" "${{ env.VERSION }}" "ap-southeast-2" "${{ vars.AWS_AMI_STAGING_BUCKET }}" timeout-minutes: 60 + + # Mirror the AMI registered above into the regions we have presence in + # (plus us-east-1 as the default consumer region). AMIs are region-scoped + # so without this consumers in other regions would have to copy-image + # themselves before launching. + copy-amis: + needs: [register-ami] + if: startsWith(github.ref, 'refs/tags/') + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + suite: [noble, resolute] + region: [ap-southeast-6, eu-central-5, ap-south-1, us-east-1] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + + - name: Derive version from tag + run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_ENV" + + # copy-image is called with both --source-region and --region, but + # configure-aws-credentials still needs a region for the session. Use + # the target region so describe-images / modify-image-attribute calls + # against the copy don't need an explicit --region argument. + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + aws-region: ${{ matrix.region }} + role-to-assume: arn:aws:iam::658427548944:role/gha-linux-images-upload + role-session-name: GHA@linux-images=CopyAMI-${{ matrix.suite }}-${{ matrix.arch }}-${{ matrix.region }} + + - name: Copy AMI to ${{ matrix.region }} + run: scripts/copy-ami-to-region.sh "${{ matrix.arch }}" "${{ matrix.suite }}" "${{ env.VERSION }}" "ap-southeast-2" "${{ matrix.region }}" + timeout-minutes: 60 diff --git a/scripts/copy-ami-to-region.sh b/scripts/copy-ami-to-region.sh new file mode 100755 index 0000000..a5f8e1f --- /dev/null +++ b/scripts/copy-ami-to-region.sh @@ -0,0 +1,182 @@ +#!/bin/bash +set -euo pipefail + +# Copy a registered AMI from the source region into a target region and +# make the copy public. +# +# AWS AMIs are region-scoped: an AMI registered in ap-southeast-2 can only +# be launched directly from ap-southeast-2. Consumers in other regions +# would otherwise have to copy-image themselves before they could launch. +# This script does that copy on their behalf for the regions we care to +# mirror to (see the copy-amis job matrix in .github/workflows/build.yml). +# +# Like register-ami-for-release.sh, the copy is made public +# (launch-permission Group=all + create-volume-permission Group=all on the +# backing snapshot). Public AMIs require unencrypted snapshots; copy-image +# preserves the source snapshot's encryption status when no key is +# specified, so as long as the source is unencrypted the copy is too. +# +# Usage: ./copy-ami-to-region.sh +# +# Arguments: +# arch Architecture: amd64 or arm64 +# suite Ubuntu suite codename: noble or resolute +# version Release version string (e.g. "1.2.3", without leading "v") +# source-region AWS region where the AMI was originally registered +# target-region AWS region to copy the AMI into + +ARCH="${1:-}" +SUITE="${2:-}" +VERSION="${3:-}" +SOURCE_REGION="${4:-}" +TARGET_REGION="${5:-}" + +if [ -z "$ARCH" ] || [ -z "$SUITE" ] || [ -z "$VERSION" ] || [ -z "$SOURCE_REGION" ] || [ -z "$TARGET_REGION" ]; then + echo "Usage: $0 " + exit 1 +fi + +case "$ARCH" in + amd64|arm64) ;; + *) echo "ERROR: arch must be amd64 or arm64, got: $ARCH"; exit 1 ;; +esac + +# Keep in lockstep with register-ami-for-release.sh. +case "$SUITE" in + noble) UBUNTU_VERSION="24.04" ;; + resolute) UBUNTU_VERSION="26.04" ;; + *) echo "ERROR: unknown suite '$SUITE' (add a mapping here and in the justfile)"; exit 1 ;; +esac + +AMI_NAME="ubuntu-${UBUNTU_VERSION}-bes-cloud-${ARCH}-${VERSION}" + +echo "Architecture : $ARCH" +echo "Suite : $SUITE ($UBUNTU_VERSION)" +echo "Version : $VERSION" +echo "Source region : $SOURCE_REGION" +echo "Target region : $TARGET_REGION" +echo "AMI name : $AMI_NAME" +echo "" + +# --- Idempotency guard --- + +EXISTING=$(aws ec2 describe-images \ + --region "$TARGET_REGION" \ + --owners self \ + --filters "Name=name,Values=${AMI_NAME}" \ + --query 'Images[0].ImageId' \ + --output text) +if [ "$EXISTING" != "None" ] && [ -n "$EXISTING" ]; then + echo "Copy already exists in $TARGET_REGION: $EXISTING — skipping." + exit 0 +fi + +# --- Resolve source AMI --- + +SOURCE_AMI=$(aws ec2 describe-images \ + --region "$SOURCE_REGION" \ + --owners self \ + --filters "Name=name,Values=${AMI_NAME}" \ + --query 'Images[0].ImageId' \ + --output text) +if [ "$SOURCE_AMI" = "None" ] || [ -z "$SOURCE_AMI" ]; then + echo "ERROR: source AMI $AMI_NAME not found in $SOURCE_REGION" + exit 1 +fi +echo "Source AMI: $SOURCE_AMI" +echo "" + +# --- Initiate copy --- + +# Tag the AMI at creation time. The backing snapshot doesn't get tagged +# from --tag-specifications=image, so we tag it explicitly once the copy +# becomes available and we know the snapshot ID. +TAGS="Key=Name,Value=${AMI_NAME} Key=Os,Value=Ubuntu Key=OsVersion,Value=${UBUNTU_VERSION} Key=OsCodename,Value=${SUITE} Key=Variant,Value=cloud Key=Architecture,Value=${ARCH} Key=Version,Value=${VERSION} Key=Features,Value=BTRFS Key=Builder,Value=BES" +TAG_SPEC="ResourceType=image,Tags=[{Key=Name,Value=${AMI_NAME}},{Key=Os,Value=Ubuntu},{Key=OsVersion,Value=${UBUNTU_VERSION}},{Key=OsCodename,Value=${SUITE}},{Key=Variant,Value=cloud},{Key=Architecture,Value=${ARCH}},{Key=Version,Value=${VERSION}},{Key=Features,Value=BTRFS},{Key=Builder,Value=BES}]" + +echo "Initiating copy-image ..." +COPY_AMI=$(aws ec2 copy-image \ + --region "$TARGET_REGION" \ + --source-region "$SOURCE_REGION" \ + --source-image-id "$SOURCE_AMI" \ + --name "$AMI_NAME" \ + --description "BES Ubuntu ${UBUNTU_VERSION} cloud ${ARCH} ${VERSION} with BTRFS" \ + --tag-specifications "$TAG_SPEC" \ + --query 'ImageId' \ + --output text) +echo "Copy AMI ID: $COPY_AMI" +echo "" + +# --- Poll until copy completes --- + +echo "Waiting for copy to become available (typically 5-20 minutes) ..." +MAX_SECS=3600 +ELAPSED=0 +while true; do + STATE=$(aws ec2 describe-images \ + --region "$TARGET_REGION" \ + --image-ids "$COPY_AMI" \ + --query 'Images[0].State' \ + --output text) + echo " [${ELAPSED}s] state=$STATE" + case "$STATE" in + available) echo ""; break ;; + failed|invalid|error|deregistered) + REASON=$(aws ec2 describe-images \ + --region "$TARGET_REGION" \ + --image-ids "$COPY_AMI" \ + --query 'Images[0].StateReason' \ + --output text) + echo "ERROR: copy entered terminal state $STATE: $REASON" + exit 1 + ;; + esac + if [ "$ELAPSED" -ge "$MAX_SECS" ]; then + echo "ERROR: timed out after ${MAX_SECS}s" + exit 1 + fi + sleep 30 + ELAPSED=$((ELAPSED + 30)) +done + +# --- Tag the backing snapshot --- + +SNAPSHOT_ID=$(aws ec2 describe-images \ + --region "$TARGET_REGION" \ + --image-ids "$COPY_AMI" \ + --query 'Images[0].BlockDeviceMappings[0].Ebs.SnapshotId' \ + --output text) +# shellcheck disable=SC2086 # $TAGS is intentionally word-split into AWS CLI tag args +aws ec2 create-tags \ + --region "$TARGET_REGION" \ + --resources "$SNAPSHOT_ID" \ + --tags $TAGS +echo "Tagged backing snapshot $SNAPSHOT_ID" +echo "" + +# --- Make AMI and snapshot public --- + +echo "Publishing AMI and snapshot ..." +aws ec2 modify-image-attribute \ + --region "$TARGET_REGION" \ + --image-id "$COPY_AMI" \ + --launch-permission 'Add=[{Group=all}]' +aws ec2 modify-snapshot-attribute \ + --region "$TARGET_REGION" \ + --snapshot-id "$SNAPSHOT_ID" \ + --create-volume-permission 'Add=[{Group=all}]' + +PUBLIC=$(aws ec2 describe-images \ + --region "$TARGET_REGION" \ + --image-ids "$COPY_AMI" \ + --query 'Images[0].Public' \ + --output text) +if [ "$PUBLIC" != "True" ]; then + echo "ERROR: copy $COPY_AMI did not become public (Public=$PUBLIC)" + exit 1 +fi +echo "Published and verified" +echo "" + +echo "Done." +echo "AMI $COPY_AMI ($AMI_NAME) is registered and public in $TARGET_REGION." From 364f27f869cf65284240a5f79b16c57d05b24d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Mon, 22 Jun 2026 06:31:23 +1200 Subject: [PATCH 2/3] ci(aws): publish AMI IDs in the release manifest and download page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release-aggregate job already builds the manifest.json + index.html that consumers see at tools.ops.tamanu.io/linux-images//. Make register-ami and copy-amis each emit a small JSON fragment with the AMI ID, region, arch, suite, and Ubuntu version of the AMI they produced (or already-found, on idempotent re-runs). release-aggregate downloads them all, includes them in manifest.json under a new `amis` field, and renders a 'Launch on AWS' table on the download page with a one-click console URL per (region × arch × suite). The job's needs now include register-ami and copy-amis to make sure the fragments are present, but copy-amis can partially fail without blocking the release — release-aggregate uses always() + explicit result checks to require only the strictly necessary jobs (image builds + register-ami). Fragments from failed satellite regions simply don't appear in the table. --- .github/workflows/build.yml | 80 +++++++++++++++++++++++++++-- scripts/copy-ami-to-region.sh | 13 +++++ scripts/register-ami-for-release.sh | 13 +++++ 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c45edad..2199f65 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -573,8 +573,21 @@ jobs: # tag; failed tests on a tagged build are an operator-side abort, same # as register-ami already presumes. release-aggregate: - needs: [images-cloud, images-metal, images-pi, pi-eeprom, iso] - if: startsWith(github.ref, 'refs/tags/') + needs: [images-cloud, images-metal, images-pi, pi-eeprom, iso, register-ami, copy-amis] + # copy-amis is in needs to force a wait, but the AMI section is built from + # whatever fragments actually upload — a partial satellite-region failure + # shouldn't block the release. always() lets us run despite a failed + # copy-amis job, while explicit result checks gate on the truly required + # jobs. + if: | + always() && + startsWith(github.ref, 'refs/tags/') && + needs.images-cloud.result == 'success' && + needs.images-metal.result == 'success' && + needs.images-pi.result == 'success' && + needs.pi-eeprom.result == 'success' && + needs.iso.result == 'success' && + needs.register-ami.result == 'success' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -588,6 +601,15 @@ jobs: pattern: 'manifest-fragment-*' path: fragments/ + - name: Download AMI fragments + # Zero or more — copy-amis matrix entries are best-effort, so some + # regions may have failed to upload. + uses: actions/download-artifact@v8 + with: + pattern: 'ami-fragment-*' + path: amis/ + continue-on-error: true + - name: Assemble manifest.json + SHA256SUMS # r[image.output.checksum] run: | mkdir -p release @@ -596,13 +618,24 @@ jobs: # sort by name for deterministic output. jq -s 'add | sort_by(.name)' fragments/*/manifest-fragment.json > files.json + # AMI fragments are single JSON objects (one per registered/copied + # AMI). Some may be missing if a copy-amis matrix entry failed. + shopt -s nullglob + AMI_FILES=(amis/*/ami-fragment.json) + if [ "${#AMI_FILES[@]}" -gt 0 ]; then + jq -s 'sort_by(.ubuntu_version, .arch, .region)' "${AMI_FILES[@]}" > amis.json + else + echo '[]' > amis.json + fi + jq -n \ --arg version "$VERSION" \ --arg date "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --arg repo "https://github.com/${{ github.repository }}" \ --arg tag "${{ github.ref_name }}" \ --slurpfile files files.json \ - '{ version: $version, date: $date, repo: $repo, tag: $tag, files: $files[0] }' \ + --slurpfile amis amis.json \ + '{ version: $version, date: $date, repo: $repo, tag: $tag, files: $files[0], amis: $amis[0] }' \ > release/manifest.json jq -r '.files[] | "\(.sha256) \(.name)"' release/manifest.json > release/SHA256SUMS @@ -642,6 +675,7 @@ jobs: TABLE_ROWS_PLACEHOLDER + AMI_SECTION_PLACEHOLDER HTMLEOF @@ -671,6 +705,24 @@ jobs: echo "${name}${variant}${suite}${arch}${format}${hsize}" done > table_rows.tmp + # AMI section: only render if there are any AMIs. The console + # quick-launch URL takes the AMI ID; the AMIs are public so any + # AWS account can hit it. + if [ "$(jq -r '.amis | length' manifest.json)" -gt 0 ]; then + { + echo '

Launch on AWS

' + echo '

These AMIs are public — use the Launch link below or run aws ec2 run-instances --image-id <ami-id> --region <region> ....

' + echo '' + echo '' + echo '' + jq -r '.amis[] | ""' manifest.json + echo '' + echo '
RegionUbuntuArchAMI ID
\(.region)\(.ubuntu_version)\(.arch)\(.ami_id)Launch
' + } > ami_section.tmp + else + : > ami_section.tmp + fi + sed -i "s|VERSION_PLACEHOLDER|${VERSION}|g" index.html sed -i "s|REPO_PLACEHOLDER|${repo}|g" index.html sed -i "s|TAG_PLACEHOLDER|${tag}|g" index.html @@ -678,7 +730,11 @@ jobs: r table_rows.tmp d }" index.html - rm -f table_rows.tmp + sed -i "/AMI_SECTION_PLACEHOLDER/{ + r ami_section.tmp + d + }" index.html + rm -f table_rows.tmp ami_section.tmp - run: ls -lh release/ @@ -778,6 +834,14 @@ jobs: run: scripts/register-ami-for-release.sh "${{ matrix.arch }}" "${{ matrix.suite }}" "${{ env.VERSION }}" "ap-southeast-2" "${{ vars.AWS_AMI_STAGING_BUCKET }}" timeout-minutes: 60 + - name: Upload AMI fragment + uses: actions/upload-artifact@v7 + with: + name: ami-fragment-ap-southeast-2-${{ matrix.suite }}-${{ matrix.arch }} + path: ami-fragment.json + if-no-files-found: error + retention-days: 1 + # Mirror the AMI registered above into the regions we have presence in # (plus us-east-1 as the default consumer region). AMIs are region-scoped # so without this consumers in other regions would have to copy-image @@ -812,3 +876,11 @@ jobs: - name: Copy AMI to ${{ matrix.region }} run: scripts/copy-ami-to-region.sh "${{ matrix.arch }}" "${{ matrix.suite }}" "${{ env.VERSION }}" "ap-southeast-2" "${{ matrix.region }}" timeout-minutes: 60 + + - name: Upload AMI fragment + uses: actions/upload-artifact@v7 + with: + name: ami-fragment-${{ matrix.region }}-${{ matrix.suite }}-${{ matrix.arch }} + path: ami-fragment.json + if-no-files-found: error + retention-days: 1 diff --git a/scripts/copy-ami-to-region.sh b/scripts/copy-ami-to-region.sh index a5f8e1f..66e3c1c 100755 --- a/scripts/copy-ami-to-region.sh +++ b/scripts/copy-ami-to-region.sh @@ -58,6 +58,16 @@ echo "Target region : $TARGET_REGION" echo "AMI name : $AMI_NAME" echo "" +# Emit a JSON fragment describing this AMI so the release-aggregate job can +# surface it on the download page. Called both on the early-exit (copy +# already exists) path and at the end of a fresh copy. +emit_fragment() { + local ami_id="$1" + cat > ami-fragment.json < ami-fragment.json < Date: Mon, 22 Jun 2026 06:39:15 +1200 Subject: [PATCH 3/3] =?UTF-8?q?ci(aws):=20fix=20eu-central-5=20=E2=86=92?= =?UTF-8?q?=20eu-central-2=20(Zurich)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eu-central-5 doesn't exist; the satellite region is Zurich (eu-central-2). --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2199f65..ccd377c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -854,7 +854,7 @@ jobs: matrix: arch: [amd64, arm64] suite: [noble, resolute] - region: [ap-southeast-6, eu-central-5, ap-south-1, us-east-1] + region: [ap-southeast-6, eu-central-2, ap-south-1, us-east-1] runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6