diff --git a/.github/workflows/build-macos-image.yml b/.github/workflows/build-macos-image.yml index d220356..68f381a 100644 --- a/.github/workflows/build-macos-image.yml +++ b/.github/workflows/build-macos-image.yml @@ -31,7 +31,10 @@ permissions: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 180 + # setup now bakes a settled image in one job (base pull + offline provision + a full + # boot through the ~40min-on-metal first-boot finalization + slim + capture + push), + # which on the nested-KVM runner can approach the old 180 ceiling — give it headroom. + timeout-minutes: 300 env: STAGE: ${{ github.event.inputs.stage }} DISK_SIZE: ${{ github.event.inputs.disk_size }} diff --git a/qemu/launch.go b/qemu/launch.go index fe0b3ab..623b9a1 100644 --- a/qemu/launch.go +++ b/qemu/launch.go @@ -1,7 +1,7 @@ // Package qemu builds and launches the qemu-system-x86_64 command for booting // macOS (Sequoia 15 / Tahoe 26) via OpenCore on an x86 Linux/KVM host. The argument vector spoofs a -// GenuineIntel Skylake-Client-v4 + isa-applesmc OSK + OVMF + an OpenCore boot disk; with the LongQT -// OpenCore EFI it boots identically on Intel and AMD hosts. +// GenuineIntel Skylake-Client (TSX off, invariant-TSC frequency fed via CPUID) + isa-applesmc OSK + +// OVMF + an OpenCore boot disk; with the LongQT OpenCore EFI it boots identically on Intel and AMD. package qemu import ( @@ -13,10 +13,19 @@ const ( // OSK is the Apple SMC key required for macOS guests (public, from OSX-KVM). OSK = "ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc" - // macOSCPU spoofs a GenuineIntel Skylake-Client-v4 — the model the LongQT OpenCore expects, which - // boots macOS identically on Intel and AMD hosts (AMD support lives in the OpenCore EFI, not the - // -cpu). kvm=on exposes the hypervisor CPUID leaf macOS reads for the invtsc frequency. - macOSCPU = "Skylake-Client-v4,vendor=GenuineIntel,kvm=on" + // macOSCPU is the -cpu for macOS Sequoia/Tahoe. Every token is load-bearing and MUST NOT be + // simplified away — a testbed proved a stripped "Skylake-Client-v4,vendor=GenuineIntel,kvm=on" + // makes a fresh image's first boot SPIN FOREVER (the "completing installation" storm never ends): + // -hle,-rtm disable TSX; the v4 model enables it and macOS spins on TSX under nested KVM + // +invtsc,vmware-cpuid-freq=on hand macOS the invariant-TSC frequency via CPUID so it skips the + // self-calibration that spins under an unstable nested-KVM TSC + // vendor=GenuineIntel macOS refuses to boot on a non-Intel vendor string + // the +flags are perf (pcid/invpcid TLB, tsc-deadline timer, …); most are in the base but are + // affirmed here with check= so a base-model bump can't silently drop them again. + // AMD support comes from the LongQT OpenCore EFI, not the -cpu, so this boots on Intel and AMD. + macOSCPU = "Skylake-Client,-hle,-rtm,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on," + + "+ssse3,+sse4.2,+popcnt,+avx,+aes,+xsave,+xsaveopt," + + "+pcid,+invpcid,+tsc-deadline,+rdtscp,+xsavec,check" // ahciDriveOpts is the shared -drive tuning for every writable macOS AHCI/IDE disk (OS disk + // data disks; macOS has no virtio-blk). io_uring beats the default threads aio backend, and diff --git a/qemu/launch_test.go b/qemu/launch_test.go index 0de0719..b8d93e7 100644 --- a/qemu/launch_test.go +++ b/qemu/launch_test.go @@ -136,10 +136,18 @@ func TestArgsCPU(t *testing.T) { if len(cpu) != 1 { t.Fatalf("-cpu count: %v", cpu) } - // the unified LongQT recipe: a GenuineIntel-spoofed Skylake-Client-v4 with the hypervisor leaf on - for _, f := range []string{"Skylake-Client-v4", "vendor=GenuineIntel", "kvm=on"} { + if !strings.HasPrefix(cpu[0], "Skylake-Client,") { + t.Fatalf("-cpu base must be Skylake-Client (v4 enables TSX -> macOS first-boot spins): %s", cpu[0]) + } + // Guard every load-bearing token so a base-model bump or "simplification" can't silently drop the + // ones a fresh image's first boot needs (TSX off + TSC-freq via CPUID); this is the regression + // dacf35c introduced by stripping the string down to Skylake-Client-v4. + for _, f := range []string{ + "vendor=GenuineIntel", "kvm=on", "-hle", "-rtm", "+invtsc", "vmware-cpuid-freq=on", + "+pcid", "+invpcid", "+tsc-deadline", "+rdtscp", + } { if !strings.Contains(cpu[0], f) { - t.Fatalf("-cpu missing %s: %s", f, cpu[0]) + t.Fatalf("-cpu missing load-bearing %s: %s", f, cpu[0]) } } } diff --git a/scripts/build-qemu-macos.sh b/scripts/build-qemu-macos.sh index eebd274..c2304a9 100755 --- a/scripts/build-qemu-macos.sh +++ b/scripts/build-qemu-macos.sh @@ -3,7 +3,8 @@ # MACOS_SHORTNAME (+ GHCR_REPO/GHCR_TAG/QCOW2_NAME) — the workflow derives them from its `macos` # input: tahoe=26 (the last Intel macOS) or sequoia=15. Boots the exact firmware stack production # uses: the LongQT OpenCore baked by scripts/lib-firmware.sh + apt ovmf (OVMF_*_4M.fd) + a -# GenuineIntel Skylake-Client-v4 CPU. kholia/OSX-KVM is cloned only for fetch-macOS-v2.py (the +# GenuineIntel Skylake-Client CPU (TSX off + invariant-TSC via CPUID; see qemu/launch.go for why every +# flag is load-bearing). kholia/OSX-KVM is cloned only for fetch-macOS-v2.py (the # Apple recovery download). # # STAGE controls how far we go (the macOS install has no autounattend, so it is @@ -194,7 +195,7 @@ launch_qemu() { log "launching QEMU headless (VNC 127.0.0.1:590$VNC_DISP, ssh :$SSH_PORT, monitor $MON_SOCK)" local args=( -enable-kvm -m "$MEMORY" - -cpu Skylake-Client-v4,vendor=GenuineIntel,kvm=on + -cpu Skylake-Client,-hle,-rtm,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,+ssse3,+sse4.2,+popcnt,+avx,+aes,+xsave,+xsaveopt,+pcid,+invpcid,+tsc-deadline,+rdtscp,+xsavec,check -machine q35 -smp "$CPUS",cores=2,sockets=1 -device qemu-xhci,id=xhci -device usb-kbd,bus=xhci.0 -device usb-tablet,bus=xhci.0 @@ -376,8 +377,49 @@ stage_setup() { # M2b: boot Recovery (keyboard works there) + provision the ins sleep 12; screendump "su-03-provision-$(printf '%02d' "$i")" kill -0 "$QEMU_PID" 2>/dev/null || { log "QEMU exited"; return 1; } done - log "provision done; capturing turnkey image -> $GHCR_REPO:$GHCR_TAG" - capture_and_push "$GHCR_TAG" # tahoe:26 = SA-skipped, user 'cocoon', SSH on first boot + log "provision done; rebooting into the installed OS to settle its first-boot finalization" + mon "quit"; sleep 10 + [[ -f "$WORKDIR/qemu.pid" ]] && kill "$(cat "$WORKDIR/qemu.pid")" 2>/dev/null || true + sleep 3 + # provisioning ran offline from Recovery, so the installed OS never did its one-time first-full-boot + # finalization (kernelcache/dyld rebuild, LaunchServices registration — a ~40min CPU/disk storm). + # Boot it here, once, at build time and bake a SETTLED image, so consumers never pay that tax. + configure_opencore hide # installed macOS becomes the sole/default picker entry + launch_qemu + absorb_finalization_and_capture "$GHCR_TAG" # tahoe:26 = SA-skipped, user 'cocoon', SSH-ready, settled +} + +# absorb_finalization_and_capture boots the attached MacHDD, waits through the one-time first-full-boot +# finalization over SSH, bakes the perf recipe, slims free space, cleanly shuts down, and captures+pushes +# the settled image as $1. Shared by stage_setup (initial bake) and stage_slim (re-slim a published tag). +absorb_finalization_and_capture() { + local tag="$1" + boot_macintosh + # Settle the one-time first-boot finalization (kernelcache/dyld rebuild) at build time so consumers + # don't. macOS 26/15 stops at the Setup Assistant, which blocks BOTH SSH (sshd resets the kex) and + # ACPI powerdown — so we can't wait on SSH or shut down cleanly. Instead wait for the guest CPU to + # go idle (finalization done), flush, then hard-stop. qemu-nbd check confirms the qcow2 stays + # consistent across the hard stop, and the settled overlay boots straight to the SA (no re-run). + log "settling first-boot finalization (waiting on CPU idle; SA blocks SSH+ACPI on macOS 26/15)" + wait_cpu_idle 200 || log "WARN: CPU never idled in budget; capturing current state anyway" + sleep 60 # let macOS flush its last writes to the qcow2 before the hard stop + log "hard-stopping the VM (no clean-shutdown path at the SA; qcow2 stays consistent)" + mon "quit"; sleep 8 + [[ -f "$WORKDIR/qemu.pid" ]] && kill "$(cat "$WORKDIR/qemu.pid")" 2>/dev/null || true + sleep 3 + capture_and_push "$tag" # convert -c compresses the settled image +} + +wait_cpu_idle() { # block until QEMU's CPU stays low (first-boot finalization finished). arg = max 20s ticks + local idle=0 c w + for w in $(seq 1 "${1:-200}"); do + c=$(top -b -n1 -p "$QEMU_PID" 2>/dev/null | awk -v p="$QEMU_PID" '$1==p{print int($9)}') + if [ "${c:-999}" -lt 40 ]; then idle=$((idle + 1)); else idle=0; fi + [ "$idle" -ge 3 ] && { log "CPU idle 3x (~$((w * 20))s) — finalization settled"; return 0; } + [ $((w % 6)) -eq 0 ] && screendump "settle-$(printf '%03d' "$w")" + sleep 20 + done + return 1 } picker_size() { stat -f%z "$ARTIFACT_DIR/$1.png" 2>/dev/null || stat -c%s "$ARTIFACT_DIR/$1.png" 2>/dev/null || echo 0; } @@ -533,20 +575,10 @@ stage_desktop() { # turn the SSH-ready :26 into a boot-straight-to-desktop + sl capture_and_push "$GHCR_TAG" # tahoe:26 = boots straight to cocoon's desktop, slimmed } -stage_slim() { # SA-INDEPENDENT slim: boot, reclaim stale clusters over SSH, re-push the same tag. - # Works on SSH-ready images (:26) regardless of the GUI being stuck at Setup Assistant — slimming - # only needs SSH (sudo+dd work at the SA stage) + the MacHDD discard=unmap,detect-zeroes=unmap. - boot_macintosh - log "waiting for SSH (slim is SA-independent; SSH comes up even with the GUI at Setup Assistant)" - # Sequoia's cold first-boot to SSH is slow on the nested-KVM GHA runner (Tahoe verify came up in - # ~minutes, but a 12-min budget flaked); give it a generous ~22 min before declaring failure. - wait_ssh 66 || { log "FATAL: SSH never came up"; return 1; } - apply_perf_recipe # bake the macOS-side perf tweaks into the image before reclaiming/repushing - slim_disk - log "clean shutdown" - gssh 'echo cocoon | sudo -S shutdown -h now' >/dev/null 2>&1 || true - sleep 20 - capture_and_push "$GHCR_TAG" # convert -c drops the now-zeroed/unmapped stale clusters +stage_slim() { # re-slim an already-published tag: boot, reclaim stale clusters over SSH, re-push. + # Now that stage_setup bakes a settled image, this is a re-run knob (e.g. to re-slim after changes); + # it shares the same boot-through-finalization + slim + clean-shutdown + capture path. + absorb_finalization_and_capture "$GHCR_TAG" } main() { diff --git a/scripts/provision-macos.sh b/scripts/provision-macos.sh index d81dae7..87290e4 100644 --- a/scripts/provision-macos.sh +++ b/scripts/provision-macos.sh @@ -26,6 +26,26 @@ for d in "$VOL/private/var/db" "$VOL/var/db"; do done ls -la "$VOL/private/var/db/.AppleSetupDone" 2>/dev/null && echo "OK .AppleSetupDone" || echo "WARN .AppleSetupDone missing" +# 1b) Exclude the Data volume from Spotlight OFFLINE. mds/diskarbitrationd honor a root-level +# .metadata_never_index at volume-mount time — before any indexing is scheduled — so this closes the +# race the RunAtLoad `mdutil -a -i off` (step 3) loses: on the heavy first full boot, Spotlight's +# whole-disk crawl is the single biggest avoidable CPU/disk hog, and mdutil only fires after the +# storm has already started. The live mdutil calls stay as a harmless fallback. +touch "$VOL/.metadata_never_index" && echo "OK .metadata_never_index (Spotlight off at mount)" + +# 1c) Disable automatic software-update checks OFFLINE so the first boot doesn't spend it phoning home. +mkdir -p "$VOL/Library/Preferences" +cat > "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist" <<'PLIST' + + + + AutomaticCheckEnabled + AutomaticDownload + +PLIST +chown 0:0 "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist"; chmod 644 "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist" +echo "OK SoftwareUpdate auto-check disabled" + # 2) create the admin user directly in the volume's dslocal U="/Local/Default/Users/$USER_NAME" dscl -f "$DS" localhost -create "$U" @@ -46,6 +66,7 @@ SYS="${VOL% - Data}" # the installed System volume is the Data volume's siblin [ -d "$v/System/Library/User Template" ] && SYS="$v" && break done echo "=== SYSTEM VOLUME = [$SYS] ===" +[ -n "$SYS" ] && touch "$SYS/.metadata_never_index" 2>/dev/null || true # Spotlight off on System vol too mkdir -p "$VOL/Users/$USER_NAME" if [ -n "$SYS" ]; then TPL="$SYS/System/Library/User Template"