Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/build-macos-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@ permissions:
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 180
# setup now bakes a settled image in one job (base pull + offline provision + a full
# boot through the ~40min-on-metal first-boot finalization + slim + capture + push),
# which on the nested-KVM runner can approach the old 180 ceiling — give it headroom.
timeout-minutes: 300
env:
STAGE: ${{ github.event.inputs.stage }}
DISK_SIZE: ${{ github.event.inputs.disk_size }}
Expand Down
21 changes: 15 additions & 6 deletions qemu/launch.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Package qemu builds and launches the qemu-system-x86_64 command for booting
// macOS (Sequoia 15 / Tahoe 26) via OpenCore on an x86 Linux/KVM host. The argument vector spoofs a
// GenuineIntel Skylake-Client-v4 + isa-applesmc OSK + OVMF + an OpenCore boot disk; with the LongQT
// OpenCore EFI it boots identically on Intel and AMD hosts.
// GenuineIntel Skylake-Client (TSX off, invariant-TSC frequency fed via CPUID) + isa-applesmc OSK +
// OVMF + an OpenCore boot disk; with the LongQT OpenCore EFI it boots identically on Intel and AMD.
package qemu

import (
Expand All @@ -13,10 +13,19 @@ const (
// OSK is the Apple SMC key required for macOS guests (public, from OSX-KVM).
OSK = "ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc"

// macOSCPU spoofs a GenuineIntel Skylake-Client-v4 — the model the LongQT OpenCore expects, which
// boots macOS identically on Intel and AMD hosts (AMD support lives in the OpenCore EFI, not the
// -cpu). kvm=on exposes the hypervisor CPUID leaf macOS reads for the invtsc frequency.
macOSCPU = "Skylake-Client-v4,vendor=GenuineIntel,kvm=on"
// macOSCPU is the -cpu for macOS Sequoia/Tahoe. Every token is load-bearing and MUST NOT be
// simplified away — a testbed proved a stripped "Skylake-Client-v4,vendor=GenuineIntel,kvm=on"
// makes a fresh image's first boot SPIN FOREVER (the "completing installation" storm never ends):
// -hle,-rtm disable TSX; the v4 model enables it and macOS spins on TSX under nested KVM
// +invtsc,vmware-cpuid-freq=on hand macOS the invariant-TSC frequency via CPUID so it skips the
// self-calibration that spins under an unstable nested-KVM TSC
// vendor=GenuineIntel macOS refuses to boot on a non-Intel vendor string
// the +flags are perf (pcid/invpcid TLB, tsc-deadline timer, …); most are in the base but are
// affirmed here with check= so a base-model bump can't silently drop them again.
// AMD support comes from the LongQT OpenCore EFI, not the -cpu, so this boots on Intel and AMD.
macOSCPU = "Skylake-Client,-hle,-rtm,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on," +
"+ssse3,+sse4.2,+popcnt,+avx,+aes,+xsave,+xsaveopt," +
"+pcid,+invpcid,+tsc-deadline,+rdtscp,+xsavec,check"

// ahciDriveOpts is the shared -drive tuning for every writable macOS AHCI/IDE disk (OS disk +
// data disks; macOS has no virtio-blk). io_uring beats the default threads aio backend, and
Expand Down
14 changes: 11 additions & 3 deletions qemu/launch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,18 @@ func TestArgsCPU(t *testing.T) {
if len(cpu) != 1 {
t.Fatalf("-cpu count: %v", cpu)
}
// the unified LongQT recipe: a GenuineIntel-spoofed Skylake-Client-v4 with the hypervisor leaf on
for _, f := range []string{"Skylake-Client-v4", "vendor=GenuineIntel", "kvm=on"} {
if !strings.HasPrefix(cpu[0], "Skylake-Client,") {
t.Fatalf("-cpu base must be Skylake-Client (v4 enables TSX -> macOS first-boot spins): %s", cpu[0])
}
// Guard every load-bearing token so a base-model bump or "simplification" can't silently drop the
// ones a fresh image's first boot needs (TSX off + TSC-freq via CPUID); this is the regression
// dacf35c introduced by stripping the string down to Skylake-Client-v4.
for _, f := range []string{
"vendor=GenuineIntel", "kvm=on", "-hle", "-rtm", "+invtsc", "vmware-cpuid-freq=on",
"+pcid", "+invpcid", "+tsc-deadline", "+rdtscp",
} {
if !strings.Contains(cpu[0], f) {
t.Fatalf("-cpu missing %s: %s", f, cpu[0])
t.Fatalf("-cpu missing load-bearing %s: %s", f, cpu[0])
}
}
}
Expand Down
68 changes: 50 additions & 18 deletions scripts/build-qemu-macos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
# MACOS_SHORTNAME (+ GHCR_REPO/GHCR_TAG/QCOW2_NAME) — the workflow derives them from its `macos`
# input: tahoe=26 (the last Intel macOS) or sequoia=15. Boots the exact firmware stack production
# uses: the LongQT OpenCore baked by scripts/lib-firmware.sh + apt ovmf (OVMF_*_4M.fd) + a
# GenuineIntel Skylake-Client-v4 CPU. kholia/OSX-KVM is cloned only for fetch-macOS-v2.py (the
# GenuineIntel Skylake-Client CPU (TSX off + invariant-TSC via CPUID; see qemu/launch.go for why every
# flag is load-bearing). kholia/OSX-KVM is cloned only for fetch-macOS-v2.py (the
# Apple recovery download).
#
# STAGE controls how far we go (the macOS install has no autounattend, so it is
Expand Down Expand Up @@ -194,7 +195,7 @@ launch_qemu() {
log "launching QEMU headless (VNC 127.0.0.1:590$VNC_DISP, ssh :$SSH_PORT, monitor $MON_SOCK)"
local args=(
-enable-kvm -m "$MEMORY"
-cpu Skylake-Client-v4,vendor=GenuineIntel,kvm=on
-cpu Skylake-Client,-hle,-rtm,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,+ssse3,+sse4.2,+popcnt,+avx,+aes,+xsave,+xsaveopt,+pcid,+invpcid,+tsc-deadline,+rdtscp,+xsavec,check
-machine q35
-smp "$CPUS",cores=2,sockets=1
-device qemu-xhci,id=xhci -device usb-kbd,bus=xhci.0 -device usb-tablet,bus=xhci.0
Expand Down Expand Up @@ -376,8 +377,49 @@ stage_setup() { # M2b: boot Recovery (keyboard works there) + provision the ins
sleep 12; screendump "su-03-provision-$(printf '%02d' "$i")"
kill -0 "$QEMU_PID" 2>/dev/null || { log "QEMU exited"; return 1; }
done
log "provision done; capturing turnkey image -> $GHCR_REPO:$GHCR_TAG"
capture_and_push "$GHCR_TAG" # tahoe:26 = SA-skipped, user 'cocoon', SSH on first boot
log "provision done; rebooting into the installed OS to settle its first-boot finalization"
mon "quit"; sleep 10
[[ -f "$WORKDIR/qemu.pid" ]] && kill "$(cat "$WORKDIR/qemu.pid")" 2>/dev/null || true
sleep 3
# provisioning ran offline from Recovery, so the installed OS never did its one-time first-full-boot
# finalization (kernelcache/dyld rebuild, LaunchServices registration — a ~40min CPU/disk storm).
# Boot it here, once, at build time and bake a SETTLED image, so consumers never pay that tax.
configure_opencore hide # installed macOS becomes the sole/default picker entry
launch_qemu
absorb_finalization_and_capture "$GHCR_TAG" # tahoe:26 = SA-skipped, user 'cocoon', SSH-ready, settled
}

# absorb_finalization_and_capture boots the attached MacHDD, waits through the one-time first-full-boot
# finalization over SSH, bakes the perf recipe, slims free space, cleanly shuts down, and captures+pushes
# the settled image as $1. Shared by stage_setup (initial bake) and stage_slim (re-slim a published tag).
absorb_finalization_and_capture() {
local tag="$1"
boot_macintosh
# Settle the one-time first-boot finalization (kernelcache/dyld rebuild) at build time so consumers
# don't. macOS 26/15 stops at the Setup Assistant, which blocks BOTH SSH (sshd resets the kex) and
# ACPI powerdown — so we can't wait on SSH or shut down cleanly. Instead wait for the guest CPU to
# go idle (finalization done), flush, then hard-stop. qemu-nbd check confirms the qcow2 stays
# consistent across the hard stop, and the settled overlay boots straight to the SA (no re-run).
log "settling first-boot finalization (waiting on CPU idle; SA blocks SSH+ACPI on macOS 26/15)"
wait_cpu_idle 200 || log "WARN: CPU never idled in budget; capturing current state anyway"
sleep 60 # let macOS flush its last writes to the qcow2 before the hard stop
log "hard-stopping the VM (no clean-shutdown path at the SA; qcow2 stays consistent)"
mon "quit"; sleep 8
[[ -f "$WORKDIR/qemu.pid" ]] && kill "$(cat "$WORKDIR/qemu.pid")" 2>/dev/null || true
sleep 3
capture_and_push "$tag" # convert -c compresses the settled image
}

wait_cpu_idle() { # block until QEMU's CPU stays low (first-boot finalization finished). arg = max 20s ticks
local idle=0 c w
for w in $(seq 1 "${1:-200}"); do
c=$(top -b -n1 -p "$QEMU_PID" 2>/dev/null | awk -v p="$QEMU_PID" '$1==p{print int($9)}')
if [ "${c:-999}" -lt 40 ]; then idle=$((idle + 1)); else idle=0; fi
[ "$idle" -ge 3 ] && { log "CPU idle 3x (~$((w * 20))s) — finalization settled"; return 0; }
[ $((w % 6)) -eq 0 ] && screendump "settle-$(printf '%03d' "$w")"
sleep 20
done
return 1
}

picker_size() { stat -f%z "$ARTIFACT_DIR/$1.png" 2>/dev/null || stat -c%s "$ARTIFACT_DIR/$1.png" 2>/dev/null || echo 0; }
Expand Down Expand Up @@ -533,20 +575,10 @@ stage_desktop() { # turn the SSH-ready :26 into a boot-straight-to-desktop + sl
capture_and_push "$GHCR_TAG" # tahoe:26 = boots straight to cocoon's desktop, slimmed
}

stage_slim() { # SA-INDEPENDENT slim: boot, reclaim stale clusters over SSH, re-push the same tag.
# Works on SSH-ready images (:26) regardless of the GUI being stuck at Setup Assistant — slimming
# only needs SSH (sudo+dd work at the SA stage) + the MacHDD discard=unmap,detect-zeroes=unmap.
boot_macintosh
log "waiting for SSH (slim is SA-independent; SSH comes up even with the GUI at Setup Assistant)"
# Sequoia's cold first-boot to SSH is slow on the nested-KVM GHA runner (Tahoe verify came up in
# ~minutes, but a 12-min budget flaked); give it a generous ~22 min before declaring failure.
wait_ssh 66 || { log "FATAL: SSH never came up"; return 1; }
apply_perf_recipe # bake the macOS-side perf tweaks into the image before reclaiming/repushing
slim_disk
log "clean shutdown"
gssh 'echo cocoon | sudo -S shutdown -h now' >/dev/null 2>&1 || true
sleep 20
capture_and_push "$GHCR_TAG" # convert -c drops the now-zeroed/unmapped stale clusters
stage_slim() { # re-slim an already-published tag: boot, reclaim stale clusters over SSH, re-push.
# Now that stage_setup bakes a settled image, this is a re-run knob (e.g. to re-slim after changes);
# it shares the same boot-through-finalization + slim + clean-shutdown + capture path.
absorb_finalization_and_capture "$GHCR_TAG"
}

main() {
Expand Down
21 changes: 21 additions & 0 deletions scripts/provision-macos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,26 @@ for d in "$VOL/private/var/db" "$VOL/var/db"; do
done
ls -la "$VOL/private/var/db/.AppleSetupDone" 2>/dev/null && echo "OK .AppleSetupDone" || echo "WARN .AppleSetupDone missing"

# 1b) Exclude the Data volume from Spotlight OFFLINE. mds/diskarbitrationd honor a root-level
# .metadata_never_index at volume-mount time — before any indexing is scheduled — so this closes the
# race the RunAtLoad `mdutil -a -i off` (step 3) loses: on the heavy first full boot, Spotlight's
# whole-disk crawl is the single biggest avoidable CPU/disk hog, and mdutil only fires after the
# storm has already started. The live mdutil calls stay as a harmless fallback.
touch "$VOL/.metadata_never_index" && echo "OK .metadata_never_index (Spotlight off at mount)"

# 1c) Disable automatic software-update checks OFFLINE so the first boot doesn't spend it phoning home.
mkdir -p "$VOL/Library/Preferences"
cat > "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist" <<'PLIST'
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0"><dict>
<key>AutomaticCheckEnabled</key><false/>
<key>AutomaticDownload</key><false/>
</dict></plist>
PLIST
chown 0:0 "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist"; chmod 644 "$VOL/Library/Preferences/com.apple.SoftwareUpdate.plist"
echo "OK SoftwareUpdate auto-check disabled"

# 2) create the admin user directly in the volume's dslocal
U="/Local/Default/Users/$USER_NAME"
dscl -f "$DS" localhost -create "$U"
Expand All @@ -46,6 +66,7 @@ SYS="${VOL% - Data}" # the installed System volume is the Data volume's siblin
[ -d "$v/System/Library/User Template" ] && SYS="$v" && break
done
echo "=== SYSTEM VOLUME = [$SYS] ==="
[ -n "$SYS" ] && touch "$SYS/.metadata_never_index" 2>/dev/null || true # Spotlight off on System vol too
mkdir -p "$VOL/Users/$USER_NAME"
if [ -n "$SYS" ]; then
TPL="$SYS/System/Library/User Template"
Expand Down
Loading