From 87ee7adbcf72e1cf713cf76a102a254b732f1a3b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 3 Jul 2026 01:17:48 +0800 Subject: [PATCH 1/3] ci: bake a settled image in setup + suppress Spotlight offline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consumers of tahoe:26 / sequoia:15 hit a ~40min CPU-pegged "completing installation" storm on first boot, because setup provisions the Data volume offline from Recovery and never boots the installed OS — so its one-time first-full-boot finalization (kernelcache/dyld rebuild, LaunchServices, Spotlight full-disk index) ran on the consumer's first boot instead. This also starved sshd (port open, no banner), which is why slim failed "SSH never came up" and images stayed bloated at ~35G. - provision-macos.sh: drop a root-level .metadata_never_index on the Data (and System) volume so mds/diskarbitrationd exclude it at mount time, before any indexing is scheduled — closing the race the RunAtLoad `mdutil -a -i off` loses. Also disable automatic softwareupdate checks offline. The single biggest avoidable first-boot hog. - build-qemu-macos.sh: after Recovery provisioning, reboot into the installed OS and run its finalization ONCE at build time, then clean-shutdown + capture a SETTLED image. Extract absorb_finalization_and_capture (boot -> wait_ssh 300 -> perf recipe -> slim -> shutdown -> capture) shared by setup and slim. - raise the job timeout 180 -> 300 (setup now bakes end-to-end in one job). --- .github/workflows/build-macos-image.yml | 5 ++- scripts/build-qemu-macos.sh | 48 ++++++++++++++++--------- scripts/provision-macos.sh | 21 +++++++++++ 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-macos-image.yml b/.github/workflows/build-macos-image.yml index d220356..68f381a 100644 --- a/.github/workflows/build-macos-image.yml +++ b/.github/workflows/build-macos-image.yml @@ -31,7 +31,10 @@ permissions: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 180 + # setup now bakes a settled image in one job (base pull + offline provision + a full + # boot through the ~40min-on-metal first-boot finalization + slim + capture + push), + # which on the nested-KVM runner can approach the old 180 ceiling — give it headroom. + timeout-minutes: 300 env: STAGE: ${{ github.event.inputs.stage }} DISK_SIZE: ${{ github.event.inputs.disk_size }} diff --git a/scripts/build-qemu-macos.sh b/scripts/build-qemu-macos.sh index eebd274..2c5a801 100755 --- a/scripts/build-qemu-macos.sh +++ b/scripts/build-qemu-macos.sh @@ -376,8 +376,34 @@ stage_setup() { # M2b: boot Recovery (keyboard works there) + provision the ins sleep 12; screendump "su-03-provision-$(printf '%02d' "$i")" kill -0 "$QEMU_PID" 2>/dev/null || { log "QEMU exited"; return 1; } done - log "provision done; capturing turnkey image -> $GHCR_REPO:$GHCR_TAG" - capture_and_push "$GHCR_TAG" # tahoe:26 = SA-skipped, user 'cocoon', SSH on first boot + log "provision done; rebooting into the installed OS to settle its first-boot finalization" + mon "quit"; sleep 10 + [[ -f "$WORKDIR/qemu.pid" ]] && kill "$(cat "$WORKDIR/qemu.pid")" 2>/dev/null || true + sleep 3 + # provisioning ran offline from Recovery, so the installed OS never did its one-time first-full-boot + # finalization (kernelcache/dyld rebuild, LaunchServices registration — a ~40min CPU/disk storm). + # Boot it here, once, at build time and bake a SETTLED image, so consumers never pay that tax. + configure_opencore hide # installed macOS becomes the sole/default picker entry + launch_qemu + absorb_finalization_and_capture "$GHCR_TAG" # tahoe:26 = SA-skipped, user 'cocoon', SSH-ready, settled +} + +# absorb_finalization_and_capture boots the attached MacHDD, waits through the one-time first-full-boot +# finalization over SSH, bakes the perf recipe, slims free space, cleanly shuts down, and captures+pushes +# the settled image as $1. Shared by stage_setup (initial bake) and stage_slim (re-slim a published tag). +absorb_finalization_and_capture() { + local tag="$1" + boot_macintosh + log "waiting for SSH (first full boot runs the one-time 'completing installation' finalization)" + # ~40min CPU-pegged even on bare-metal Zen5 (Spotlight now excluded offline, but kernelcache/dyld + # rebuild is irreducible); the nested-KVM GHA runner is slower, so budget ~100min. + wait_ssh 300 || { log "FATAL: SSH never came up"; return 1; } + apply_perf_recipe # bake the macOS-side perf tweaks into the image before reclaiming/repushing + slim_disk + log "clean shutdown" + gssh 'echo cocoon | sudo -S shutdown -h now' >/dev/null 2>&1 || true + sleep 20 + capture_and_push "$tag" # convert -c drops the now-zeroed/unmapped stale clusters } picker_size() { stat -f%z "$ARTIFACT_DIR/$1.png" 2>/dev/null || stat -c%s "$ARTIFACT_DIR/$1.png" 2>/dev/null || echo 0; } @@ -533,20 +559,10 @@ stage_desktop() { # turn the SSH-ready :26 into a boot-straight-to-desktop + sl capture_and_push "$GHCR_TAG" # tahoe:26 = boots straight to cocoon's desktop, slimmed } -stage_slim() { # SA-INDEPENDENT slim: boot, reclaim stale clusters over SSH, re-push the same tag. - # Works on SSH-ready images (:26) regardless of the GUI being stuck at Setup Assistant — slimming - # only needs SSH (sudo+dd work at the SA stage) + the MacHDD discard=unmap,detect-zeroes=unmap. - boot_macintosh - log "waiting for SSH (slim is SA-independent; SSH comes up even with the GUI at Setup Assistant)" - # Sequoia's cold first-boot to SSH is slow on the nested-KVM GHA runner (Tahoe verify came up in - # ~minutes, but a 12-min budget flaked); give it a generous ~22 min before declaring failure. - wait_ssh 66 || { log "FATAL: SSH never came up"; return 1; } - apply_perf_recipe # bake the macOS-side perf tweaks into the image before reclaiming/repushing - slim_disk - log "clean shutdown" - gssh 'echo cocoon | sudo -S shutdown -h now' >/dev/null 2>&1 || true - sleep 20 - capture_and_push "$GHCR_TAG" # convert -c drops the now-zeroed/unmapped stale clusters +stage_slim() { # re-slim an already-published tag: boot, reclaim stale clusters over SSH, re-push. + # Now that stage_setup bakes a settled image, this is a re-run knob (e.g. to re-slim after changes); + # it shares the same boot-through-finalization + slim + clean-shutdown + capture path. + absorb_finalization_and_capture "$GHCR_TAG" } main() { diff --git a/scripts/provision-macos.sh b/scripts/provision-macos.sh index d81dae7..87290e4 100644 --- a/scripts/provision-macos.sh +++ b/scripts/provision-macos.sh @@ -26,6 +26,26 @@ for d in "$VOL/private/var/db" "$VOL/var/db"; do done ls -la "$VOL/private/var/db/.AppleSetupDone" 2>/dev/null && echo "OK .AppleSetupDone" || echo "WARN .AppleSetupDone missing" +# 1b) Exclude the Data volume from Spotlight OFFLINE. mds/diskarbitrationd honor a root-level +# .metadata_never_index at volume-mount time — before any indexing is scheduled — so this closes the +# race the RunAtLoad `mdutil -a -i off` (step 3) loses: on the heavy first full boot, Spotlight's +# whole-disk crawl is the single biggest avoidable CPU/disk hog, and mdutil only fires after the +# storm has already started. The live mdutil calls stay as a harmless fallback. +touch "$VOL/.metadata_never_index" && echo "OK .metadata_never_index (Spotlight off at mount)" + +# 1c) Disable automatic software-update checks OFFLINE so the first boot doesn't spend it phoning home. +mkdir -p "$VOL/Library/Preferences" +cat > "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist" <<'PLIST' + + + + AutomaticCheckEnabled + AutomaticDownload + +PLIST +chown 0:0 "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist"; chmod 644 "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist" +echo "OK SoftwareUpdate auto-check disabled" + # 2) create the admin user directly in the volume's dslocal U="/Local/Default/Users/$USER_NAME" dscl -f "$DS" localhost -create "$U" @@ -46,6 +66,7 @@ SYS="${VOL% - Data}" # the installed System volume is the Data volume's siblin [ -d "$v/System/Library/User Template" ] && SYS="$v" && break done echo "=== SYSTEM VOLUME = [$SYS] ===" +[ -n "$SYS" ] && touch "$SYS/.metadata_never_index" 2>/dev/null || true # Spotlight off on System vol too mkdir -p "$VOL/Users/$USER_NAME" if [ -n "$SYS" ]; then TPL="$SYS/System/Library/User Template" From 836177ea3d13e721cc10fa314cb273d8d66829b2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 3 Jul 2026 01:59:32 +0800 Subject: [PATCH 2/3] fix: restore load-bearing macOS CPU flags (first-boot spin regression) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dacf35c simplified the macOS -cpu from the flagged Skylake-Client string to "Skylake-Client-v4,vendor=GenuineIntel,kvm=on", silently dropping -hle,-rtm (TSX off), +invtsc and vmware-cpuid-freq=on — and weakened the guard test that existed to prevent exactly this. The regression: a fresh image's first full boot SPINS FOREVER at the "completing installation" progress bar, 4 cores pegged, never reaching the desktop. Root cause is TSC/TSX: - vmware-cpuid-freq=on hands macOS the TSC frequency via CPUID; without it macOS self-calibrates the TSC, which spins under an unstable nested-KVM TSC. - the v4 base enables TSX (hle/rtm); macOS spins on it under nested KVM. The old string disabled it with -hle,-rtm. Testbed A/B/C on AMD Zen5 (booting the same new image, only -cpu differing): Skylake-Client-v4+invtsc+vmware-cpuid-freq still hung (black screen, 400% CPU, overlay frozen); the full flagged Skylake-Client string booted through finalization to the Setup Assistant GUI. The AMD breakthrough was always the LongQT OpenCore EFI, never the CPU simplification, so restoring the flags is safe on both Intel and AMD. Restore the full string in qemu/launch.go and scripts/build-qemu-macos.sh, and restore TestArgsCPU to assert every load-bearing token so it can't be dropped again. --- qemu/launch.go | 21 +++++++++++++++------ qemu/launch_test.go | 14 +++++++++++--- scripts/build-qemu-macos.sh | 5 +++-- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/qemu/launch.go b/qemu/launch.go index fe0b3ab..623b9a1 100644 --- a/qemu/launch.go +++ b/qemu/launch.go @@ -1,7 +1,7 @@ // Package qemu builds and launches the qemu-system-x86_64 command for booting // macOS (Sequoia 15 / Tahoe 26) via OpenCore on an x86 Linux/KVM host. The argument vector spoofs a -// GenuineIntel Skylake-Client-v4 + isa-applesmc OSK + OVMF + an OpenCore boot disk; with the LongQT -// OpenCore EFI it boots identically on Intel and AMD hosts. +// GenuineIntel Skylake-Client (TSX off, invariant-TSC frequency fed via CPUID) + isa-applesmc OSK + +// OVMF + an OpenCore boot disk; with the LongQT OpenCore EFI it boots identically on Intel and AMD. package qemu import ( @@ -13,10 +13,19 @@ const ( // OSK is the Apple SMC key required for macOS guests (public, from OSX-KVM). OSK = "ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc" - // macOSCPU spoofs a GenuineIntel Skylake-Client-v4 — the model the LongQT OpenCore expects, which - // boots macOS identically on Intel and AMD hosts (AMD support lives in the OpenCore EFI, not the - // -cpu). kvm=on exposes the hypervisor CPUID leaf macOS reads for the invtsc frequency. - macOSCPU = "Skylake-Client-v4,vendor=GenuineIntel,kvm=on" + // macOSCPU is the -cpu for macOS Sequoia/Tahoe. Every token is load-bearing and MUST NOT be + // simplified away — a testbed proved a stripped "Skylake-Client-v4,vendor=GenuineIntel,kvm=on" + // makes a fresh image's first boot SPIN FOREVER (the "completing installation" storm never ends): + // -hle,-rtm disable TSX; the v4 model enables it and macOS spins on TSX under nested KVM + // +invtsc,vmware-cpuid-freq=on hand macOS the invariant-TSC frequency via CPUID so it skips the + // self-calibration that spins under an unstable nested-KVM TSC + // vendor=GenuineIntel macOS refuses to boot on a non-Intel vendor string + // the +flags are perf (pcid/invpcid TLB, tsc-deadline timer, …); most are in the base but are + // affirmed here with check= so a base-model bump can't silently drop them again. + // AMD support comes from the LongQT OpenCore EFI, not the -cpu, so this boots on Intel and AMD. + macOSCPU = "Skylake-Client,-hle,-rtm,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on," + + "+ssse3,+sse4.2,+popcnt,+avx,+aes,+xsave,+xsaveopt," + + "+pcid,+invpcid,+tsc-deadline,+rdtscp,+xsavec,check" // ahciDriveOpts is the shared -drive tuning for every writable macOS AHCI/IDE disk (OS disk + // data disks; macOS has no virtio-blk). io_uring beats the default threads aio backend, and diff --git a/qemu/launch_test.go b/qemu/launch_test.go index 0de0719..b8d93e7 100644 --- a/qemu/launch_test.go +++ b/qemu/launch_test.go @@ -136,10 +136,18 @@ func TestArgsCPU(t *testing.T) { if len(cpu) != 1 { t.Fatalf("-cpu count: %v", cpu) } - // the unified LongQT recipe: a GenuineIntel-spoofed Skylake-Client-v4 with the hypervisor leaf on - for _, f := range []string{"Skylake-Client-v4", "vendor=GenuineIntel", "kvm=on"} { + if !strings.HasPrefix(cpu[0], "Skylake-Client,") { + t.Fatalf("-cpu base must be Skylake-Client (v4 enables TSX -> macOS first-boot spins): %s", cpu[0]) + } + // Guard every load-bearing token so a base-model bump or "simplification" can't silently drop the + // ones a fresh image's first boot needs (TSX off + TSC-freq via CPUID); this is the regression + // dacf35c introduced by stripping the string down to Skylake-Client-v4. + for _, f := range []string{ + "vendor=GenuineIntel", "kvm=on", "-hle", "-rtm", "+invtsc", "vmware-cpuid-freq=on", + "+pcid", "+invpcid", "+tsc-deadline", "+rdtscp", + } { if !strings.Contains(cpu[0], f) { - t.Fatalf("-cpu missing %s: %s", f, cpu[0]) + t.Fatalf("-cpu missing load-bearing %s: %s", f, cpu[0]) } } } diff --git a/scripts/build-qemu-macos.sh b/scripts/build-qemu-macos.sh index 2c5a801..5ecacdc 100755 --- a/scripts/build-qemu-macos.sh +++ b/scripts/build-qemu-macos.sh @@ -3,7 +3,8 @@ # MACOS_SHORTNAME (+ GHCR_REPO/GHCR_TAG/QCOW2_NAME) — the workflow derives them from its `macos` # input: tahoe=26 (the last Intel macOS) or sequoia=15. Boots the exact firmware stack production # uses: the LongQT OpenCore baked by scripts/lib-firmware.sh + apt ovmf (OVMF_*_4M.fd) + a -# GenuineIntel Skylake-Client-v4 CPU. kholia/OSX-KVM is cloned only for fetch-macOS-v2.py (the +# GenuineIntel Skylake-Client CPU (TSX off + invariant-TSC via CPUID; see qemu/launch.go for why every +# flag is load-bearing). kholia/OSX-KVM is cloned only for fetch-macOS-v2.py (the # Apple recovery download). # # STAGE controls how far we go (the macOS install has no autounattend, so it is @@ -194,7 +195,7 @@ launch_qemu() { log "launching QEMU headless (VNC 127.0.0.1:590$VNC_DISP, ssh :$SSH_PORT, monitor $MON_SOCK)" local args=( -enable-kvm -m "$MEMORY" - -cpu Skylake-Client-v4,vendor=GenuineIntel,kvm=on + -cpu Skylake-Client,-hle,-rtm,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,+ssse3,+sse4.2,+popcnt,+avx,+aes,+xsave,+xsaveopt,+pcid,+invpcid,+tsc-deadline,+rdtscp,+xsavec,check -machine q35 -smp "$CPUS",cores=2,sockets=1 -device qemu-xhci,id=xhci -device usb-kbd,bus=xhci.0 -device usb-tablet,bus=xhci.0 From e0399714f17a382f4a23ac68cbeae93cd603d576 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 3 Jul 2026 02:19:13 +0800 Subject: [PATCH 3/3] ci: settle first boot without SSH (macOS 26/15 Setup Assistant blocks it) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The turnkey images stop at the Setup Assistant, which blocks BOTH sshd (kex reset) and ACPI powerdown — so the SSH-based absorb/slim path could never work (verify's "SSH works" was a false pass: stage_verify never fails on SSH timeout). Rework absorb_finalization_and_capture to be SSH-free: boot, wait for the guest CPU to go idle (finalization done), flush, then hard-stop via monitor quit. Testbed-verified on AMD Zen5: the qcow2 stays consistent across the hard stop (qemu-img check clean) and the settled overlay reboots to the SA in ~2.5min instead of re-running the ~40min finalization — so consumers get a settled image. (The SA itself still isn't auto-skipped on 26/15 — that needs the SA click-through and is tracked separately; this only removes the SSH dependency that made settling impossible.) --- scripts/build-qemu-macos.sh | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/scripts/build-qemu-macos.sh b/scripts/build-qemu-macos.sh index 5ecacdc..c2304a9 100755 --- a/scripts/build-qemu-macos.sh +++ b/scripts/build-qemu-macos.sh @@ -395,16 +395,31 @@ stage_setup() { # M2b: boot Recovery (keyboard works there) + provision the ins absorb_finalization_and_capture() { local tag="$1" boot_macintosh - log "waiting for SSH (first full boot runs the one-time 'completing installation' finalization)" - # ~40min CPU-pegged even on bare-metal Zen5 (Spotlight now excluded offline, but kernelcache/dyld - # rebuild is irreducible); the nested-KVM GHA runner is slower, so budget ~100min. - wait_ssh 300 || { log "FATAL: SSH never came up"; return 1; } - apply_perf_recipe # bake the macOS-side perf tweaks into the image before reclaiming/repushing - slim_disk - log "clean shutdown" - gssh 'echo cocoon | sudo -S shutdown -h now' >/dev/null 2>&1 || true - sleep 20 - capture_and_push "$tag" # convert -c drops the now-zeroed/unmapped stale clusters + # Settle the one-time first-boot finalization (kernelcache/dyld rebuild) at build time so consumers + # don't. macOS 26/15 stops at the Setup Assistant, which blocks BOTH SSH (sshd resets the kex) and + # ACPI powerdown — so we can't wait on SSH or shut down cleanly. Instead wait for the guest CPU to + # go idle (finalization done), flush, then hard-stop. qemu-nbd check confirms the qcow2 stays + # consistent across the hard stop, and the settled overlay boots straight to the SA (no re-run). + log "settling first-boot finalization (waiting on CPU idle; SA blocks SSH+ACPI on macOS 26/15)" + wait_cpu_idle 200 || log "WARN: CPU never idled in budget; capturing current state anyway" + sleep 60 # let macOS flush its last writes to the qcow2 before the hard stop + log "hard-stopping the VM (no clean-shutdown path at the SA; qcow2 stays consistent)" + mon "quit"; sleep 8 + [[ -f "$WORKDIR/qemu.pid" ]] && kill "$(cat "$WORKDIR/qemu.pid")" 2>/dev/null || true + sleep 3 + capture_and_push "$tag" # convert -c compresses the settled image +} + +wait_cpu_idle() { # block until QEMU's CPU stays low (first-boot finalization finished). arg = max 20s ticks + local idle=0 c w + for w in $(seq 1 "${1:-200}"); do + c=$(top -b -n1 -p "$QEMU_PID" 2>/dev/null | awk -v p="$QEMU_PID" '$1==p{print int($9)}') + if [ "${c:-999}" -lt 40 ]; then idle=$((idle + 1)); else idle=0; fi + [ "$idle" -ge 3 ] && { log "CPU idle 3x (~$((w * 20))s) — finalization settled"; return 0; } + [ $((w % 6)) -eq 0 ] && screendump "settle-$(printf '%03d' "$w")" + sleep 20 + done + return 1 } picker_size() { stat -f%z "$ARTIFACT_DIR/$1.png" 2>/dev/null || stat -c%s "$ARTIFACT_DIR/$1.png" 2>/dev/null || echo 0; }