ci(aws): mirror AMIs to satellite regions and publish IDs on download page

.github/workflows/build.yml

@@ -573,8 +573,21 @@ jobs:
   # tag; failed tests on a tagged build are an operator-side abort, same
   # as register-ami already presumes.
   release-aggregate:
-    needs: [images-cloud, images-metal, images-pi, pi-eeprom, iso]
-    if: startsWith(github.ref, 'refs/tags/')
+    needs: [images-cloud, images-metal, images-pi, pi-eeprom, iso, register-ami, copy-amis]
+    # copy-amis is in needs to force a wait, but the AMI section is built from
+    # whatever fragments actually upload — a partial satellite-region failure
+    # shouldn't block the release. always() lets us run despite a failed
+    # copy-amis job, while explicit result checks gate on the truly required
+    # jobs.
+    if: |
+      always() &&
+      startsWith(github.ref, 'refs/tags/') &&
+      needs.images-cloud.result == 'success' &&
+      needs.images-metal.result == 'success' &&
+      needs.images-pi.result == 'success' &&
+      needs.pi-eeprom.result == 'success' &&
+      needs.iso.result == 'success' &&
+      needs.register-ami.result == 'success'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -588,6 +601,15 @@ jobs:
           pattern: 'manifest-fragment-*'
           path: fragments/
 
+      - name: Download AMI fragments
+        # Zero or more — copy-amis matrix entries are best-effort, so some
+        # regions may have failed to upload.
+        uses: actions/download-artifact@v8
+        with:
+          pattern: 'ami-fragment-*'
+          path: amis/
+        continue-on-error: true
+
       - name: Assemble manifest.json + SHA256SUMS # r[image.output.checksum]
         run: |
           mkdir -p release
@@ -596,13 +618,24 @@ jobs:
           # sort by name for deterministic output.
           jq -s 'add | sort_by(.name)' fragments/*/manifest-fragment.json > files.json
 
+          # AMI fragments are single JSON objects (one per registered/copied
+          # AMI). Some may be missing if a copy-amis matrix entry failed.
+          shopt -s nullglob
+          AMI_FILES=(amis/*/ami-fragment.json)
+          if [ "${#AMI_FILES[@]}" -gt 0 ]; then
+            jq -s 'sort_by(.ubuntu_version, .arch, .region)' "${AMI_FILES[@]}" > amis.json
+          else
+            echo '[]' > amis.json
+          fi
+
           jq -n \
             --arg version "$VERSION" \
             --arg date "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
             --arg repo "https://github.com/${{ github.repository }}" \
             --arg tag "${{ github.ref_name }}" \
             --slurpfile files files.json \
-            '{ version: $version, date: $date, repo: $repo, tag: $tag, files: $files[0] }' \
+            --slurpfile amis amis.json \
+            '{ version: $version, date: $date, repo: $repo, tag: $tag, files: $files[0], amis: $amis[0] }' \
             > release/manifest.json
 
           jq -r '.files[] | "\(.sha256)  \(.name)"' release/manifest.json > release/SHA256SUMS
@@ -642,6 +675,7 @@ jobs:
           TABLE_ROWS_PLACEHOLDER
           </tbody>
           </table>
+          AMI_SECTION_PLACEHOLDER
           </body>
           </html>
           HTMLEOF
@@ -671,14 +705,36 @@ jobs:
             echo "<tr><td><a href=\"${name}\">${name}</a></td><td>${variant}</td><td>${suite}</td><td>${arch}</td><td>${format}</td><td class=\"size\">${hsize}</td></tr>"
           done > table_rows.tmp
 
+          # AMI section: only render if there are any AMIs. The console
+          # quick-launch URL takes the AMI ID; the AMIs are public so any
+          # AWS account can hit it.
+          if [ "$(jq -r '.amis | length' manifest.json)" -gt 0 ]; then
+            {
+              echo '<h2 style="margin-top:2rem;">Launch on AWS</h2>'
+              echo '<p>These AMIs are public &mdash; use the Launch link below or run <code>aws ec2 run-instances --image-id &lt;ami-id&gt; --region &lt;region&gt; ...</code>.</p>'
+              echo '<table>'
+              echo '<thead><tr><th>Region</th><th>Ubuntu</th><th>Arch</th><th>AMI ID</th><th></th></tr></thead>'
+              echo '<tbody>'
+              jq -r '.amis[] | "<tr><td>\(.region)</td><td>\(.ubuntu_version)</td><td>\(.arch)</td><td><code>\(.ami_id)</code></td><td><a href=\"https://\(.region).console.aws.amazon.com/ec2/home?region=\(.region)#LaunchInstances:ami=\(.ami_id)\">Launch</a></td></tr>"' manifest.json
+              echo '</tbody>'
+              echo '</table>'
+            } > ami_section.tmp
+          else
+            : > ami_section.tmp
+          fi
+
           sed -i "s|VERSION_PLACEHOLDER|${VERSION}|g" index.html
           sed -i "s|REPO_PLACEHOLDER|${repo}|g" index.html
           sed -i "s|TAG_PLACEHOLDER|${tag}|g" index.html
           sed -i "/TABLE_ROWS_PLACEHOLDER/{
             r table_rows.tmp
             d
           }" index.html
-          rm -f table_rows.tmp
+          sed -i "/AMI_SECTION_PLACEHOLDER/{
+            r ami_section.tmp
+            d
+          }" index.html
+          rm -f table_rows.tmp ami_section.tmp
 
       - run: ls -lh release/
 
@@ -777,3 +833,54 @@ jobs:
         # every time the bucket is recreated.
         run: scripts/register-ami-for-release.sh "${{ matrix.arch }}" "${{ matrix.suite }}" "${{ env.VERSION }}" "ap-southeast-2" "${{ vars.AWS_AMI_STAGING_BUCKET }}"
         timeout-minutes: 60
+
+      - name: Upload AMI fragment
+        uses: actions/upload-artifact@v7
+        with:
+          name: ami-fragment-ap-southeast-2-${{ matrix.suite }}-${{ matrix.arch }}
+          path: ami-fragment.json
+          if-no-files-found: error
+          retention-days: 1
+
+  # Mirror the AMI registered above into the regions we have presence in
+  # (plus us-east-1 as the default consumer region). AMIs are region-scoped
+  # so without this consumers in other regions would have to copy-image
+  # themselves before launching.
+  copy-amis:
+    needs: [register-ami]
+    if: startsWith(github.ref, 'refs/tags/')
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [amd64, arm64]
+        suite: [noble, resolute]
+        region: [ap-southeast-6, eu-central-2, ap-south-1, us-east-1]
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Derive version from tag
+        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_ENV"
+
+      # copy-image is called with both --source-region and --region, but
+      # configure-aws-credentials still needs a region for the session. Use
+      # the target region so describe-images / modify-image-attribute calls
+      # against the copy don't need an explicit --region argument.
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          aws-region: ${{ matrix.region }}
+          role-to-assume: arn:aws:iam::658427548944:role/gha-linux-images-upload
+          role-session-name: GHA@linux-images=CopyAMI-${{ matrix.suite }}-${{ matrix.arch }}-${{ matrix.region }}
+
+      - name: Copy AMI to ${{ matrix.region }}
+        run: scripts/copy-ami-to-region.sh "${{ matrix.arch }}" "${{ matrix.suite }}" "${{ env.VERSION }}" "ap-southeast-2" "${{ matrix.region }}"
+        timeout-minutes: 60
+
+      - name: Upload AMI fragment
+        uses: actions/upload-artifact@v7
+        with:
+          name: ami-fragment-${{ matrix.region }}-${{ matrix.suite }}-${{ matrix.arch }}
+          path: ami-fragment.json
+          if-no-files-found: error
+          retention-days: 1

scripts/copy-ami-to-region.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+set -euo pipefail
+
+# Copy a registered AMI from the source region into a target region and
+# make the copy public.
+#
+# AWS AMIs are region-scoped: an AMI registered in ap-southeast-2 can only
+# be launched directly from ap-southeast-2. Consumers in other regions
+# would otherwise have to copy-image themselves before they could launch.
+# This script does that copy on their behalf for the regions we care to
+# mirror to (see the copy-amis job matrix in .github/workflows/build.yml).
+#
+# Like register-ami-for-release.sh, the copy is made public
+# (launch-permission Group=all + create-volume-permission Group=all on the
+# backing snapshot). Public AMIs require unencrypted snapshots; copy-image
+# preserves the source snapshot's encryption status when no key is
+# specified, so as long as the source is unencrypted the copy is too.
+#
+# Usage: ./copy-ami-to-region.sh <arch> <suite> <version> <source-region> <target-region>
+#
+# Arguments:
+#   arch           Architecture: amd64 or arm64
+#   suite          Ubuntu suite codename: noble or resolute
+#   version        Release version string (e.g. "1.2.3", without leading "v")
+#   source-region  AWS region where the AMI was originally registered
+#   target-region  AWS region to copy the AMI into
+
+ARCH="${1:-}"
+SUITE="${2:-}"
+VERSION="${3:-}"
+SOURCE_REGION="${4:-}"
+TARGET_REGION="${5:-}"
+
+if [ -z "$ARCH" ] || [ -z "$SUITE" ] || [ -z "$VERSION" ] || [ -z "$SOURCE_REGION" ] || [ -z "$TARGET_REGION" ]; then
+    echo "Usage: $0 <arch> <suite> <version> <source-region> <target-region>"
+    exit 1
+fi
+
+case "$ARCH" in
+    amd64|arm64) ;;
+    *) echo "ERROR: arch must be amd64 or arm64, got: $ARCH"; exit 1 ;;
+esac
+
+# Keep in lockstep with register-ami-for-release.sh.
+case "$SUITE" in
+    noble)    UBUNTU_VERSION="24.04" ;;
+    resolute) UBUNTU_VERSION="26.04" ;;
+    *) echo "ERROR: unknown suite '$SUITE' (add a mapping here and in the justfile)"; exit 1 ;;
+esac
+
+AMI_NAME="ubuntu-${UBUNTU_VERSION}-bes-cloud-${ARCH}-${VERSION}"
+
+echo "Architecture  : $ARCH"
+echo "Suite         : $SUITE ($UBUNTU_VERSION)"
+echo "Version       : $VERSION"
+echo "Source region : $SOURCE_REGION"
+echo "Target region : $TARGET_REGION"
+echo "AMI name      : $AMI_NAME"
+echo ""
+
+# Emit a JSON fragment describing this AMI so the release-aggregate job can
+# surface it on the download page. Called both on the early-exit (copy
+# already exists) path and at the end of a fresh copy.
+emit_fragment() {
+    local ami_id="$1"
+    cat > ami-fragment.json <<EOF
+{"region":"$TARGET_REGION","arch":"$ARCH","suite":"$SUITE","ubuntu_version":"$UBUNTU_VERSION","version":"$VERSION","ami_id":"$ami_id","name":"$AMI_NAME"}
+EOF
+}
+
+# --- Idempotency guard ---
+
+EXISTING=$(aws ec2 describe-images \
+    --region "$TARGET_REGION" \
+    --owners self \
+    --filters "Name=name,Values=${AMI_NAME}" \
+    --query 'Images[0].ImageId' \
+    --output text)
+if [ "$EXISTING" != "None" ] && [ -n "$EXISTING" ]; then
+    echo "Copy already exists in $TARGET_REGION: $EXISTING — skipping."
+    emit_fragment "$EXISTING"
+    exit 0
+fi
+
+# --- Resolve source AMI ---
+
+SOURCE_AMI=$(aws ec2 describe-images \
+    --region "$SOURCE_REGION" \
+    --owners self \
+    --filters "Name=name,Values=${AMI_NAME}" \
+    --query 'Images[0].ImageId' \
+    --output text)
+if [ "$SOURCE_AMI" = "None" ] || [ -z "$SOURCE_AMI" ]; then
+    echo "ERROR: source AMI $AMI_NAME not found in $SOURCE_REGION"
+    exit 1
+fi
+echo "Source AMI: $SOURCE_AMI"
+echo ""
+
+# --- Initiate copy ---
+
+# Tag the AMI at creation time. The backing snapshot doesn't get tagged
+# from --tag-specifications=image, so we tag it explicitly once the copy
+# becomes available and we know the snapshot ID.
+TAGS="Key=Name,Value=${AMI_NAME} Key=Os,Value=Ubuntu Key=OsVersion,Value=${UBUNTU_VERSION} Key=OsCodename,Value=${SUITE} Key=Variant,Value=cloud Key=Architecture,Value=${ARCH} Key=Version,Value=${VERSION} Key=Features,Value=BTRFS Key=Builder,Value=BES"
+TAG_SPEC="ResourceType=image,Tags=[{Key=Name,Value=${AMI_NAME}},{Key=Os,Value=Ubuntu},{Key=OsVersion,Value=${UBUNTU_VERSION}},{Key=OsCodename,Value=${SUITE}},{Key=Variant,Value=cloud},{Key=Architecture,Value=${ARCH}},{Key=Version,Value=${VERSION}},{Key=Features,Value=BTRFS},{Key=Builder,Value=BES}]"
+
+echo "Initiating copy-image ..."
+COPY_AMI=$(aws ec2 copy-image \
+    --region "$TARGET_REGION" \
+    --source-region "$SOURCE_REGION" \
+    --source-image-id "$SOURCE_AMI" \
+    --name "$AMI_NAME" \
+    --description "BES Ubuntu ${UBUNTU_VERSION} cloud ${ARCH} ${VERSION} with BTRFS" \
+    --tag-specifications "$TAG_SPEC" \
+    --query 'ImageId' \
+    --output text)
+echo "Copy AMI ID: $COPY_AMI"
+echo ""
+
+# --- Poll until copy completes ---
+
+echo "Waiting for copy to become available (typically 5-20 minutes) ..."
+MAX_SECS=3600
+ELAPSED=0
+while true; do
+    STATE=$(aws ec2 describe-images \
+        --region "$TARGET_REGION" \
+        --image-ids "$COPY_AMI" \
+        --query 'Images[0].State' \
+        --output text)
+    echo "  [${ELAPSED}s] state=$STATE"
+    case "$STATE" in
+        available) echo ""; break ;;
+        failed|invalid|error|deregistered)
+            REASON=$(aws ec2 describe-images \
+                --region "$TARGET_REGION" \
+                --image-ids "$COPY_AMI" \
+                --query 'Images[0].StateReason' \
+                --output text)
+            echo "ERROR: copy entered terminal state $STATE: $REASON"
+            exit 1
+            ;;
+    esac
+    if [ "$ELAPSED" -ge "$MAX_SECS" ]; then
+        echo "ERROR: timed out after ${MAX_SECS}s"
+        exit 1
+    fi
+    sleep 30
+    ELAPSED=$((ELAPSED + 30))
+done
+
+# --- Tag the backing snapshot ---
+
+SNAPSHOT_ID=$(aws ec2 describe-images \
+    --region "$TARGET_REGION" \
+    --image-ids "$COPY_AMI" \
+    --query 'Images[0].BlockDeviceMappings[0].Ebs.SnapshotId' \
+    --output text)
+# shellcheck disable=SC2086 # $TAGS is intentionally word-split into AWS CLI tag args
+aws ec2 create-tags \
+    --region "$TARGET_REGION" \
+    --resources "$SNAPSHOT_ID" \
+    --tags $TAGS
+echo "Tagged backing snapshot $SNAPSHOT_ID"
+echo ""
+
+# --- Make AMI and snapshot public ---
+
+echo "Publishing AMI and snapshot ..."
+aws ec2 modify-image-attribute \
+    --region "$TARGET_REGION" \
+    --image-id "$COPY_AMI" \
+    --launch-permission 'Add=[{Group=all}]'
+aws ec2 modify-snapshot-attribute \
+    --region "$TARGET_REGION" \
+    --snapshot-id "$SNAPSHOT_ID" \
+    --create-volume-permission 'Add=[{Group=all}]'
+
+PUBLIC=$(aws ec2 describe-images \
+    --region "$TARGET_REGION" \
+    --image-ids "$COPY_AMI" \
+    --query 'Images[0].Public' \
+    --output text)
+if [ "$PUBLIC" != "True" ]; then
+    echo "ERROR: copy $COPY_AMI did not become public (Public=$PUBLIC)"
+    exit 1
+fi
+echo "Published and verified"
+echo ""
+
+emit_fragment "$COPY_AMI"
+
+echo "Done."
+echo "AMI $COPY_AMI ($AMI_NAME) is registered and public in $TARGET_REGION."

scripts/register-ami-for-release.sh

@@ -61,6 +61,16 @@ echo "S3 bucket    : $BUCKET"
 echo "AMI name     : $AMI_NAME"
 echo ""
 
+# Emit a JSON fragment describing this AMI so the release-aggregate job can
+# surface it on the download page. Called both on the early-exit
+# (already-registered) path and at the end of a fresh registration.
+emit_fragment() {
+    local ami_id="$1"
+    cat > ami-fragment.json <<EOF
+{"region":"$REGION","arch":"$ARCH","suite":"$SUITE","ubuntu_version":"$UBUNTU_VERSION","version":"$VERSION","ami_id":"$ami_id","name":"$AMI_NAME"}
+EOF
+}
+
 # Check whether an AMI with this name already exists (idempotency guard).
 EXISTING=$(aws ec2 describe-images \
     --region "$REGION" \
@@ -70,6 +80,7 @@ EXISTING=$(aws ec2 describe-images \
     --output text)
 if [ "$EXISTING" != "None" ] && [ -n "$EXISTING" ]; then
     echo "AMI already exists: $EXISTING — skipping registration."
+    emit_fragment "$EXISTING"
     exit 0
 fi
 
@@ -242,5 +253,7 @@ fi
 echo "Published and verified"
 echo ""
 
+emit_fragment "$AMI_ID"
+
 echo "Done."
 echo "AMI $AMI_ID ($AMI_NAME) is registered and public."