From 6221b18949b15b2d1b419dd4bf625f89859b2c71 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Apr 2026 08:09:51 +0000 Subject: [PATCH] refactor(test): run L5 E2E on GH Actions macOS runner, drop Tart VMs Cirrus CI was gated on `only_if: $CIRRUS_CRON` and the repo has no paid credits, so the ~900 lines of Tart-based L5 tests have never run. Replace the VM wrapper with a MacHost helper that exec's directly against the host, and trigger it from the free GitHub Actions macos-latest runner on release tags / workflow_dispatch. - testutil/tartvm.go -> testutil/machost.go: drop SSH/scp/expect-over-SSH and the tart clone/boot/destroy lifecycle; require CI=true or OPENBOOT_E2E_DESTRUCTIVE=1 to activate so local `go test -tags=e2e,vm` is a no-op rather than a foot-gun. - test/e2e/: rename TartVM -> MacHost across call sites; relax bare-system assumptions that no longer hold on a GH runner (brew is preinstalled, common CLI tools may be); make vmInstallHomebrew idempotent. - Drop .cirrus.yml; add macos-e2e workflow job gated on release tags / workflow_dispatch. - Makefile: trim the test-vm-release -run regex to tests that actually exist; reword comments to reflect destructive-host semantics. - CLAUDE.md / CONTRIBUTING.md: update L5 references. https://claude.ai/code/session_01Dg8nibLFZGKmeYgfUBQrsA --- .cirrus.yml | 84 --------- .github/workflows/test.yml | 18 ++ CLAUDE.md | 4 +- CONTRIBUTING.md | 4 +- Makefile | 24 +-- test/e2e/misc_e2e_test.go | 2 +- test/e2e/openboot_e2e_test.go | 14 +- test/e2e/sync_shell_e2e_test.go | 4 +- test/e2e/vm_edge_cases_test.go | 2 +- test/e2e/vm_helpers_test.go | 33 ++-- test/e2e/vm_infra_test.go | 18 +- test/e2e/vm_interactive_test.go | 2 +- test/e2e/vm_user_journey_test.go | 18 +- testutil/machost.go | 126 +++++++++++++ testutil/tartvm.go | 298 ------------------------------- 15 files changed, 209 insertions(+), 442 deletions(-) delete mode 100644 .cirrus.yml create mode 100644 testutil/machost.go delete mode 100644 testutil/tartvm.go diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index 891d931..0000000 --- a/.cirrus.yml +++ /dev/null @@ -1,84 +0,0 @@ -# Cirrus CI — VM-based E2E tests using Tart (native macOS virtualization) -# -# Cirrus CI runs on Apple Silicon and supports Tart natively, enabling -# parallel macOS VM tests that aren't possible on GitHub Actions. -# -# Trigger: manual dispatch or release tags. -# For PR testing, use GitHub Actions (unit + integration tests). - -env: - HOMEBREW_NO_AUTO_UPDATE: 1 - HOMEBREW_NO_INSTALL_CLEANUP: 1 - -# ============================================================================= -# L2: Release validation — runs on every tag push -# ============================================================================= -release_validation_task: - name: "E2E Release Validation" - only_if: $CIRRUS_CRON != '' # manual cron only, disabled for push/release - macos_instance: - image: ghcr.io/cirruslabs/macos-sequoia-base:latest - install_deps_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - brew install go hudochenkov/sshpass/sshpass cirruslabs/cli/tart - build_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - make build - test_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - make test-vm-release - timeout_in: 45m - -# ============================================================================= -# L3: Full validation — runs on release tags -# ============================================================================= -# Split into parallel tasks to maximize throughput. -# Each task runs a subset of tests in its own VM. - -full_journey_task: - name: "E2E Journey Tests" - only_if: $CIRRUS_CRON != '' # manual cron only, disabled for push/release - macos_instance: - image: ghcr.io/cirruslabs/macos-sequoia-base:latest - install_deps_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - brew install go hudochenkov/sshpass/sshpass cirruslabs/cli/tart - build_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - make build - test_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - go test -v -timeout 60m -tags="e2e,vm" -run "TestVM_Journey" ./test/e2e/... - timeout_in: 90m - -full_edge_cases_task: - name: "E2E Edge Case Tests" - only_if: $CIRRUS_CRON != '' # manual cron only, disabled for push/release - macos_instance: - image: ghcr.io/cirruslabs/macos-sequoia-base:latest - install_deps_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - brew install go hudochenkov/sshpass/sshpass cirruslabs/cli/tart - build_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - make build - test_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - go test -v -timeout 60m -tags="e2e,vm" -run "TestVM_Edge" ./test/e2e/... - timeout_in: 90m - -full_commands_task: - name: "E2E Command + Interactive Tests" - only_if: $CIRRUS_CRON != '' # manual cron only, disabled for push/release - macos_instance: - image: ghcr.io/cirruslabs/macos-sequoia-base:latest - install_deps_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - brew install go hudochenkov/sshpass/sshpass cirruslabs/cli/tart - build_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - make build - test_script: - - eval "$(/opt/homebrew/bin/brew shellenv)" - - go test -v -timeout 60m -tags="e2e,vm" -run "TestVM_Cmd|TestVM_Install|TestVM_Interactive|TestVM_Lifecycle" ./test/e2e/... - timeout_in: 90m diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c23e68..3d104c7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -158,6 +158,24 @@ jobs: if: always() run: kill $(cat /tmp/mock-pid) 2>/dev/null || true + macos-e2e: + name: macos e2e (L5) + runs-on: macos-latest + # Only on release tags or manual dispatch — these are slow and destructive. + if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: "go.mod" + + - name: Run macOS E2E (release tier) + run: make test-vm-release + cli-compat: name: old-cli compat runs-on: macos-latest diff --git a/CLAUDE.md b/CLAUDE.md index 95702cc..c632fc2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,7 @@ make build-release VERSION=0.25.0 # optimized + UPX make test-unit # L1 (~15s) — pre-push hook make test-integration # L2 (~75s) — real brew/git/npm in temp dirs make test-e2e # L4 compiled binary -make test-vm-release # L5 VM (~20m) — before tagging +make test-vm-release # L5 destructive macOS (~20m) — before tagging make test-destructive # L6 — actually installs make test-coverage # coverage.out + coverage.html @@ -60,7 +60,7 @@ internal/ ui/ # bubbletea Model pattern, lipgloss styling updater/ # Auto-update: check GitHub → download → replace test/{integration,e2e}/ # //go:build integration | e2e (+ vm, destructive, smoke) -testutil/ # shared helpers + Tart VM helpers +testutil/ # shared helpers + MacHost (destructive E2E on real macOS) scripts/ install.sh # curl|bash installer hooks/ # pre-commit, pre-push (install via `make install-hooks`) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2108ff8..8123bde 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,14 +38,14 @@ Tests are split across six tiers. Which one runs where: | **L2 Integration** | Real `brew` / `git` / `npm` against temp dirs; real `httptest` servers | `make test-integration` (~75s) | CI on push/PR | | **L3 Contract schema** | JSON schema validation against [openboot-contract](https://github.com/openbootdotdev/openboot-contract) | (runs in CI only) | CI on push/PR | | **L4 E2E binary** | Compiled binary driven by scripts; `-tags=e2e` | `make test-e2e` | CI on release | -| **L5 E2E VM** | [Tart](https://github.com/cirruslabs/tart) macOS VMs (install Homebrew, run real flows) | `make test-vm-quick` (2 min) / `test-vm-release` (20 min) / `test-vm-full` (60 min) | Manual, before tagging a release | +| **L5 Destructive macOS** | Runs against a real macOS host (installs packages, modifies `~/.zshrc`, writes `defaults`) | `make test-vm-quick` / `test-vm-release` / `test-vm-full` — requires `CI=true` or `OPENBOOT_E2E_DESTRUCTIVE=1` | GH Actions `macos-latest` on release tags + manual dispatch | | **L6 Destructive** | Actually installs real packages into a real system | `make test-destructive` / `test-smoke` | CI on release, plus manual `workflow_dispatch` | Rules of thumb: - **Local dev:** run nothing manually if hooks are installed. `make test-unit` on demand when you want a sanity check. Skip L2+ unless you're touching code that interacts with real brew/git/npm. - **Before pushing:** `make test-unit` (the pre-push hook does this automatically). -- **Before tagging a release:** `make test-vm-release` locally (needs Tart). +- **Before tagging a release:** trigger the `macos-e2e` job via GitHub Actions (manual dispatch or tag push). To run locally on a throwaway macOS machine: `OPENBOOT_E2E_DESTRUCTIVE=1 make test-vm-release`. ## Git Hooks diff --git a/Makefile b/Makefile index 27ff04e..fab850c 100644 --- a/Makefile +++ b/Makefile @@ -44,31 +44,35 @@ test-all: $(MAKE) test-coverage # ============================================================================= -# VM-based E2E tests (Tart VMs) — three levels +# Destructive macOS E2E tests — three levels # ============================================================================= +# +# These tests install real packages and modify ~/.zshrc / macOS defaults on +# the host they run on. They are intended for ephemeral macOS CI runners +# (GitHub Actions macos-latest) or a throwaway VM. +# +# On a developer machine `go test -tags="e2e,vm"` will skip unless you set +# OPENBOOT_E2E_DESTRUCTIVE=1 (see testutil/machost.go). Don't set that +# unless you mean it. -# L1: Quick validation (~2min) — run after code changes -# Runs TestVM_Infra only: boots a VM and checks SSH/arch/tools, no package installs +# L1: Quick sanity (~1min) — host/arch checks only, no package installs test-vm-quick: build go test -v -timeout 5m -tags="e2e,vm" -run "TestVM_Infra" ./test/e2e/... -# L2: Release validation (~20min) — run before tagging a release -# Core user journeys: dry-run safety, install + verify, diff/clean cycle, -# manual uninstall recovery, full setup, error messages +# L2: Release validation (~20min) — core user journeys test-vm-release: build go test -v -timeout 30m -tags="e2e,vm" \ - -run "TestVM_Infra|TestVM_Journey_DryRun|TestVM_Journey_FirstTimeUser|TestVM_Journey_ManualUninstall|TestVM_Journey_DiffConsistency|TestVM_Journey_FullSetup|TestVM_Journey_ErrorMessages" \ + -run "TestVM_Infra|TestVM_Journey_DryRunIsCompletelySafe|TestVM_Journey_FirstTimeUser|TestVM_Journey_FullSetupConfiguresEverything|TestE2E_DryRunMinimal|TestE2E_SnapshotCapture" \ ./test/e2e/... -# L3: Full validation (~60min) — run for major releases or CI -# All 48 tests: journeys + edge cases + commands + interactive +# L3: Full validation (~60min) — everything under -tags="e2e,vm" test-vm-full: build go test -v -timeout 90m -tags="e2e,vm" ./test/e2e/... # Aliases test-vm: test-vm-release -# Single VM test by name (e.g. make test-vm-run TEST=TestVM_Journey_DryRun) +# Single test by name (e.g. make test-vm-run TEST=TestVM_Journey_DryRunIsCompletelySafe) test-vm-run: build go test -v -timeout 45m -tags="e2e,vm" -run $(TEST) ./test/e2e/... diff --git a/test/e2e/misc_e2e_test.go b/test/e2e/misc_e2e_test.go index e39b2ed..8815fa2 100644 --- a/test/e2e/misc_e2e_test.go +++ b/test/e2e/misc_e2e_test.go @@ -14,7 +14,7 @@ func TestE2E_FullPreset_DryRun(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) diff --git a/test/e2e/openboot_e2e_test.go b/test/e2e/openboot_e2e_test.go index 685e5a2..3234fb0 100644 --- a/test/e2e/openboot_e2e_test.go +++ b/test/e2e/openboot_e2e_test.go @@ -17,7 +17,7 @@ func TestE2E_DryRunMinimal(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) @@ -35,7 +35,7 @@ func TestE2E_DryRunDeveloper(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) @@ -53,7 +53,7 @@ func TestE2E_SnapshotCapture(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) @@ -71,7 +71,7 @@ func TestE2E_InvalidPreset(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) @@ -91,7 +91,7 @@ func TestE2E_MissingGitConfig(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) @@ -108,7 +108,7 @@ func TestE2E_SnapshotWithOutput(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) @@ -127,7 +127,7 @@ func TestE2E_Diff_ThenClean_DryRun_SameSnapshot(t *testing.T) { } // Verify diff and clean produce consistent results from the same snapshot - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) diff --git a/test/e2e/sync_shell_e2e_test.go b/test/e2e/sync_shell_e2e_test.go index 71df9d2..390de1a 100644 --- a/test/e2e/sync_shell_e2e_test.go +++ b/test/e2e/sync_shell_e2e_test.go @@ -18,7 +18,7 @@ func TestE2E_Sync_Shell_CaptureShell(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) installOhMyZsh(t, vm) bin := vmCopyDevBinary(t, vm) @@ -49,7 +49,7 @@ func TestE2E_Sync_Shell_NoPanic(t *testing.T) { t.Skip("skipping VM test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) installOhMyZsh(t, vm) bin := vmCopyDevBinary(t, vm) diff --git a/test/e2e/vm_edge_cases_test.go b/test/e2e/vm_edge_cases_test.go index 4a248f3..171d954 100644 --- a/test/e2e/vm_edge_cases_test.go +++ b/test/e2e/vm_edge_cases_test.go @@ -27,7 +27,7 @@ func TestVM_Edge_ShellActuallyWorks(t *testing.T) { t.Skip("skipping VM edge case in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) diff --git a/test/e2e/vm_helpers_test.go b/test/e2e/vm_helpers_test.go index 066ebeb..4733098 100644 --- a/test/e2e/vm_helpers_test.go +++ b/test/e2e/vm_helpers_test.go @@ -17,7 +17,7 @@ const brewPath = "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/ // vmInstallViaBrewTap installs Homebrew and openboot via brew tap in the VM. // This mirrors the real `curl | bash` user journey. // Returns the installed openboot version string. -func vmInstallViaBrewTap(t *testing.T, vm *testutil.TartVM) string { +func vmInstallViaBrewTap(t *testing.T, vm *testutil.MacHost) string { t.Helper() script := strings.Join([]string{ @@ -36,10 +36,17 @@ func vmInstallViaBrewTap(t *testing.T, vm *testutil.TartVM) string { return strings.TrimSpace(version) } -// vmInstallHomebrew installs only Homebrew in the VM (no openboot). -func vmInstallHomebrew(t *testing.T, vm *testutil.TartVM) { +// vmInstallHomebrew ensures Homebrew is installed on the host. +// GitHub Actions macOS runners ship with Homebrew preinstalled, so this +// skips the install when brew is already on PATH and only bootstraps it +// on a genuinely bare host. +func vmInstallHomebrew(t *testing.T, vm *testutil.MacHost) { t.Helper() + if _, err := vm.Run("command -v brew >/dev/null 2>&1"); err == nil { + return + } + script := strings.Join([]string{ "export NONINTERACTIVE=1", `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"`, @@ -54,7 +61,7 @@ func vmInstallHomebrew(t *testing.T, vm *testutil.TartVM) { // vmCopyDevBinary builds the openboot binary on the host and copies it to the VM. // Returns the remote binary path. -func vmCopyDevBinary(t *testing.T, vm *testutil.TartVM) string { +func vmCopyDevBinary(t *testing.T, vm *testutil.MacHost) string { t.Helper() binary := testutil.BuildTestBinary(t) @@ -70,21 +77,21 @@ func vmCopyDevBinary(t *testing.T, vm *testutil.TartVM) string { } // vmRunOpenboot runs an openboot command inside the VM with standard PATH and env. -func vmRunOpenboot(t *testing.T, vm *testutil.TartVM, args string) (string, error) { +func vmRunOpenboot(t *testing.T, vm *testutil.MacHost, args string) (string, error) { t.Helper() cmd := fmt.Sprintf("export PATH=%q && openboot %s", brewPath, args) return vm.Run(cmd) } // vmRunDevBinary runs the dev binary inside the VM with standard PATH and env. -func vmRunDevBinary(t *testing.T, vm *testutil.TartVM, binaryPath, args string) (string, error) { +func vmRunDevBinary(t *testing.T, vm *testutil.MacHost, binaryPath, args string) (string, error) { t.Helper() cmd := fmt.Sprintf("export PATH=%q && %s %s", brewPath, binaryPath, args) return vm.Run(cmd) } // vmRunOpenbootWithGit runs openboot with git identity env vars set. -func vmRunOpenbootWithGit(t *testing.T, vm *testutil.TartVM, args string) (string, error) { +func vmRunOpenbootWithGit(t *testing.T, vm *testutil.MacHost, args string) (string, error) { t.Helper() env := map[string]string{ "PATH": brewPath, @@ -95,7 +102,7 @@ func vmRunOpenbootWithGit(t *testing.T, vm *testutil.TartVM, args string) (strin } // vmRunDevBinaryWithGit runs the dev binary with git identity env vars set. -func vmRunDevBinaryWithGit(t *testing.T, vm *testutil.TartVM, binaryPath, args string) (string, error) { +func vmRunDevBinaryWithGit(t *testing.T, vm *testutil.MacHost, binaryPath, args string) (string, error) { t.Helper() env := map[string]string{ "PATH": brewPath, @@ -106,7 +113,7 @@ func vmRunDevBinaryWithGit(t *testing.T, vm *testutil.TartVM, binaryPath, args s } // installOhMyZsh installs Oh-My-Zsh non-interactively in the VM. -func installOhMyZsh(t *testing.T, vm *testutil.TartVM) { +func installOhMyZsh(t *testing.T, vm *testutil.MacHost) { t.Helper() script := `sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended` output, err := vm.Run(script) @@ -115,7 +122,7 @@ func installOhMyZsh(t *testing.T, vm *testutil.TartVM) { } // vmBrewList returns the list of installed Homebrew formulae in the VM. -func vmBrewList(t *testing.T, vm *testutil.TartVM) []string { +func vmBrewList(t *testing.T, vm *testutil.MacHost) []string { t.Helper() output, err := vm.Run(fmt.Sprintf("export PATH=%q && brew list --formula -1 2>/dev/null", brewPath)) if err != nil { @@ -133,7 +140,7 @@ func vmBrewList(t *testing.T, vm *testutil.TartVM) []string { } // vmBrewCaskList returns the list of installed Homebrew casks in the VM. -func vmBrewCaskList(t *testing.T, vm *testutil.TartVM) []string { +func vmBrewCaskList(t *testing.T, vm *testutil.MacHost) []string { t.Helper() output, err := vm.Run(fmt.Sprintf("export PATH=%q && brew list --cask -1 2>/dev/null", brewPath)) if err != nil { @@ -151,7 +158,7 @@ func vmBrewCaskList(t *testing.T, vm *testutil.TartVM) []string { } // vmIsInstalled checks if a command is available in the VM's PATH. -func vmIsInstalled(t *testing.T, vm *testutil.TartVM, cmd string) bool { +func vmIsInstalled(t *testing.T, vm *testutil.MacHost, cmd string) bool { t.Helper() _, err := vm.Run(fmt.Sprintf("export PATH=%q && which %s", brewPath, cmd)) return err == nil @@ -163,7 +170,7 @@ func writeFile(path, content string) error { } // vmWriteTestSnapshot writes a minimal valid snapshot JSON to a path on the VM. -func vmWriteTestSnapshot(t *testing.T, vm *testutil.TartVM, remotePath string, formulae, casks, npm []string) { +func vmWriteTestSnapshot(t *testing.T, vm *testutil.MacHost, remotePath string, formulae, casks, npm []string) { t.Helper() toJSON := func(ss []string) string { diff --git a/test/e2e/vm_infra_test.go b/test/e2e/vm_infra_test.go index 03a8084..e10bd27 100644 --- a/test/e2e/vm_infra_test.go +++ b/test/e2e/vm_infra_test.go @@ -11,16 +11,16 @@ import ( "github.com/stretchr/testify/require" ) -// TestVM_Infra validates the VM infrastructure pipeline: -// clone OCI image → boot → SSH → run commands → destroy. -// This must pass before any other VM test can be trusted. +// TestVM_Infra sanity-checks the host the E2E suite runs on: we can shell +// out, we're on darwin/arm64, openboot isn't leaking in from a prior run, +// and the tools install.sh depends on (curl, git) are available. func TestVM_Infra(t *testing.T) { - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) t.Run("echo", func(t *testing.T) { - output, err := vm.Run("echo hello-from-vm") + output, err := vm.Run("echo hello-from-host") require.NoError(t, err) - assert.Contains(t, strings.TrimSpace(output), "hello-from-vm") + assert.Contains(t, strings.TrimSpace(output), "hello-from-host") }) t.Run("macos_version", func(t *testing.T) { @@ -37,12 +37,6 @@ func TestVM_Infra(t *testing.T) { assert.Contains(t, output, "arm64") }) - t.Run("no_brew_preinstalled", func(t *testing.T) { - output, err := vm.Run("which brew 2>/dev/null || echo no-brew") - require.NoError(t, err) - assert.Contains(t, output, "no-brew", "fresh VM should not have Homebrew") - }) - t.Run("no_openboot_preinstalled", func(t *testing.T) { output, err := vm.Run("which openboot 2>/dev/null || echo no-openboot") require.NoError(t, err) diff --git a/test/e2e/vm_interactive_test.go b/test/e2e/vm_interactive_test.go index e03d146..cfa759a 100644 --- a/test/e2e/vm_interactive_test.go +++ b/test/e2e/vm_interactive_test.go @@ -17,7 +17,7 @@ func TestVM_Interactive_InstallScript(t *testing.T) { t.Skip("skipping VM interactive test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallViaBrewTap(t, vm) // Install first t.Run("reinstall_answer_no", func(t *testing.T) { diff --git a/test/e2e/vm_user_journey_test.go b/test/e2e/vm_user_journey_test.go index 34e028a..28b30ba 100644 --- a/test/e2e/vm_user_journey_test.go +++ b/test/e2e/vm_user_journey_test.go @@ -37,15 +37,15 @@ func TestVM_Journey_FirstTimeUser(t *testing.T) { t.Skip("skipping full journey test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) - // Step 1: Bare system — openboot and brew should not be there - // Note: base image may have some tools preinstalled (e.g., jq in /usr/bin) + // Step 1: openboot shouldn't leak in from a prior step. We don't assert on + // rg/fd/bat/fzf absence anymore — GitHub Actions runners vary in what + // ships preinstalled, and the post-install checks below are the + // load-bearing assertion. t.Run("bare_system_has_no_openboot", func(t *testing.T) { - for _, tool := range []string{"openboot", "rg", "fd", "bat", "fzf"} { - out, _ := vm.Run("which " + tool + " 2>/dev/null || echo not-found") - assert.Contains(t, out, "not-found", "%s should not exist on bare VM", tool) - } + out, _ := vm.Run("which openboot 2>/dev/null || echo not-found") + assert.Contains(t, out, "not-found", "openboot should not exist before install") }) // Step 2: Install via curl | bash (the real user journey) @@ -96,7 +96,7 @@ func TestVM_Journey_DryRunIsCompletelySafe(t *testing.T) { t.Skip("skipping dry-run safety test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) @@ -165,7 +165,7 @@ func TestVM_Journey_FullSetupConfiguresEverything(t *testing.T) { t.Skip("skipping full setup test in short mode") } - vm := testutil.NewTartVM(t) + vm := testutil.NewMacHost(t) vmInstallHomebrew(t, vm) bin := vmCopyDevBinary(t, vm) diff --git a/testutil/machost.go b/testutil/machost.go new file mode 100644 index 0000000..d859549 --- /dev/null +++ b/testutil/machost.go @@ -0,0 +1,126 @@ +//go:build e2e && vm + +// MacHost runs destructive openboot E2E tests directly against the current +// macOS host — no Tart VM, no SSH. It's intended for ephemeral CI runners +// (GitHub Actions macos-latest) that can be thrown away after the run. +// +// A host refuses to activate unless CI=true or OPENBOOT_E2E_DESTRUCTIVE=1 +// is set, so `go test -tags="e2e,vm"` on a developer machine is a no-op +// rather than a foot-gun. + +package testutil + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +// MacHost wraps the current macOS host for an E2E test. +type MacHost struct { + t *testing.T + tempDir string +} + +// NewMacHost returns a handle on the current host. It skips the test +// unless the environment has opted in to destructive execution. +func NewMacHost(t *testing.T) *MacHost { + t.Helper() + requireEphemeralHost(t) + requireMacOS(t) + + return &MacHost{ + t: t, + tempDir: t.TempDir(), + } +} + +// Run executes a shell command and returns combined output. +func (h *MacHost) Run(command string) (string, error) { + cmd := exec.Command("bash", "-c", command) + output, err := cmd.CombinedOutput() + return string(output), err +} + +// RunWithEnv executes a command with additional environment variables. +func (h *MacHost) RunWithEnv(env map[string]string, command string) (string, error) { + cmd := exec.Command("bash", "-c", command) + cmd.Env = os.Environ() + for k, v := range env { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + output, err := cmd.CombinedOutput() + return string(output), err +} + +// ExpectStep describes one interaction step: wait for a pattern, then send input. +type ExpectStep struct { + Expect string + Send string +} + +// RunInteractive drives a TUI command via `expect(1)`. Each step waits for +// its Expect pattern, then sends Send. +func (h *MacHost) RunInteractive(command string, steps []ExpectStep, timeoutSec int) (string, error) { + if timeoutSec == 0 { + timeoutSec = 300 + } + + var script strings.Builder + fmt.Fprintf(&script, "set timeout %d\n", timeoutSec) + fmt.Fprintf(&script, "spawn bash -c %s\n", shellescape(command)) + for _, step := range steps { + fmt.Fprintf(&script, "expect %q\n", step.Expect) + fmt.Fprintf(&script, "send %q\n", step.Send) + } + script.WriteString("expect eof\n") + + tmpFile := filepath.Join(h.tempDir, fmt.Sprintf("interact-%d.exp", time.Now().UnixNano())) + if err := os.WriteFile(tmpFile, []byte(script.String()), 0644); err != nil { + return "", fmt.Errorf("write expect script: %w", err) + } + + cmd := exec.Command("expect", tmpFile) + output, err := cmd.CombinedOutput() + return string(output), err +} + +// CopyFile copies a local file to a destination path on the same host. +// Preserved for API compatibility with the old Tart-based helper. +func (h *MacHost) CopyFile(src, dst string) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("read source: %w", err) + } + if err := os.WriteFile(dst, data, 0755); err != nil { + return fmt.Errorf("write destination: %w", err) + } + return nil +} + +// Destroy is a no-op — the CI runner is the sandbox. +func (h *MacHost) Destroy() {} + +func shellescape(s string) string { + return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" +} + +func requireEphemeralHost(t *testing.T) { + t.Helper() + if os.Getenv("CI") == "true" || os.Getenv("OPENBOOT_E2E_DESTRUCTIVE") == "1" { + return + } + t.Skip("destructive macOS E2E tests require CI=true or OPENBOOT_E2E_DESTRUCTIVE=1") +} + +func requireMacOS(t *testing.T) { + t.Helper() + if runtime.GOOS != "darwin" { + t.Skip("macOS E2E tests require darwin host") + } +} diff --git a/testutil/tartvm.go b/testutil/tartvm.go deleted file mode 100644 index bec7cd4..0000000 --- a/testutil/tartvm.go +++ /dev/null @@ -1,298 +0,0 @@ -//go:build e2e && vm - -package testutil - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" -) - -const ( - ociImage = "ghcr.io/cirruslabs/macos-tahoe-base:latest" - vmUser = "admin" - vmPassword = "admin" - vmPrefix = "openboot-e2e-" - sshTimeout = 180 * time.Second - sshPollInterval = 5 * time.Second -) - -// TartVM manages the lifecycle of a Tart virtual machine for E2E testing. -type TartVM struct { - Name string - IP string - User string - sshKeyDir string - t *testing.T - destroyed bool -} - -// NewTartVM clones a fresh VM from the OCI base image, starts it, and waits for SSH. -// The VM is automatically destroyed when the test completes (via t.Cleanup). -func NewTartVM(t *testing.T) *TartVM { - t.Helper() - - requireTart(t) - requireSSHPass(t) - - name := fmt.Sprintf("%s%d", vmPrefix, time.Now().UnixNano()) - - vm := &TartVM{ - Name: name, - User: vmUser, - t: t, - } - - // Always clean up, even on panic - t.Cleanup(vm.Destroy) - - t.Logf("cloning VM %s from %s", name, ociImage) - runCmd(t, "tart", "clone", ociImage, name) - - t.Logf("starting VM %s (headless)", name) - startCmd := exec.Command("tart", "run", "--no-graphics", name) - if err := startCmd.Start(); err != nil { - t.Fatalf("failed to start VM: %v", err) - } - - vm.waitForIP(t) - vm.setupSSHKey(t) - vm.waitForSSH(t) - - return vm -} - -// Run executes a command inside the VM via SSH and returns combined output. -func (vm *TartVM) Run(command string) (string, error) { - args := vm.sshArgs() - args = append(args, vm.sshTarget(), command) - - cmd := exec.Command("ssh", args...) - output, err := cmd.CombinedOutput() - return string(output), err -} - -// RunWithEnv executes a command with environment variables set. -func (vm *TartVM) RunWithEnv(env map[string]string, command string) (string, error) { - var exports []string - for k, v := range env { - exports = append(exports, fmt.Sprintf("export %s=%q", k, v)) - } - exports = append(exports, command) - return vm.Run(strings.Join(exports, " && ")) -} - -// ExpectStep describes one interaction step: wait for a pattern, then send input. -type ExpectStep struct { - Expect string // text pattern to wait for - Send string // text to send (e.g., "\r" for Enter, "y\r" for y+Enter) -} - -// RunInteractive executes a command inside the VM using expect for TUI interaction. -// Each step waits for the Expect pattern, then sends the Send string. -// Returns the full session output. -func (vm *TartVM) RunInteractive(command string, steps []ExpectStep, timeoutSec int) (string, error) { - if timeoutSec == 0 { - timeoutSec = 300 - } - - keyPath := filepath.Join(vm.sshKeyDir, "id_ed25519") - - // Build expect script - var script strings.Builder - script.WriteString(fmt.Sprintf("set timeout %d\n", timeoutSec)) - // -t forces TTY allocation on the remote side, required for TUI apps (huh/bubbletea). - script.WriteString(fmt.Sprintf( - "spawn ssh -t -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR %s %s\n", - keyPath, vm.sshTarget(), shellescape(command), - )) - - for _, step := range steps { - script.WriteString(fmt.Sprintf("expect %q\n", step.Expect)) - script.WriteString(fmt.Sprintf("send %q\n", step.Send)) - } - - script.WriteString("expect eof\n") - - // Write expect script to temp file - tmpFile := filepath.Join(vm.sshKeyDir, "interact.exp") - if err := os.WriteFile(tmpFile, []byte(script.String()), 0644); err != nil { - return "", fmt.Errorf("failed to write expect script: %w", err) - } - - cmd := exec.Command("expect", tmpFile) - output, err := cmd.CombinedOutput() - return string(output), err -} - -// shellescape wraps a string in single quotes for shell safety. -func shellescape(s string) string { - return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" -} - -// CopyFile copies a local file into the VM via scp. -func (vm *TartVM) CopyFile(localPath, remotePath string) error { - args := vm.scpArgs() - args = append(args, localPath, fmt.Sprintf("%s@%s:%s", vm.User, vm.IP, remotePath)) - - cmd := exec.Command("scp", args...) - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("scp failed: %w, output: %s", err, string(output)) - } - return nil -} - -// Destroy stops and deletes the VM. Safe to call multiple times. -func (vm *TartVM) Destroy() { - if vm.destroyed { - return - } - vm.destroyed = true - vm.t.Logf("destroying VM %s", vm.Name) - - stop := exec.Command("tart", "stop", vm.Name) - _ = stop.Run() // ignore error if already stopped - - del := exec.Command("tart", "delete", vm.Name) - if output, err := del.CombinedOutput(); err != nil { - vm.t.Logf("warning: failed to delete VM %s: %v, output: %s", vm.Name, err, string(output)) - } -} - -func (vm *TartVM) waitForIP(t *testing.T) { - t.Helper() - t.Logf("waiting for VM IP...") - - deadline := time.Now().Add(sshTimeout) - for time.Now().Before(deadline) { - cmd := exec.Command("tart", "ip", vm.Name) - output, err := cmd.Output() - if err == nil { - ip := strings.TrimSpace(string(output)) - if ip != "" { - vm.IP = ip - t.Logf("VM IP: %s", ip) - return - } - } - time.Sleep(sshPollInterval) - } - t.Fatalf("timed out waiting for VM IP after %v", sshTimeout) -} - -func (vm *TartVM) setupSSHKey(t *testing.T) { - t.Helper() - - vm.sshKeyDir = t.TempDir() - keyPath := filepath.Join(vm.sshKeyDir, "id_ed25519") - - // Generate ephemeral SSH key pair - genCmd := exec.Command("ssh-keygen", "-t", "ed25519", "-f", keyPath, "-N", "", "-C", "openboot-vm-test") - if output, err := genCmd.CombinedOutput(); err != nil { - t.Fatalf("failed to generate SSH key: %v, output: %s", err, string(output)) - } - - // Copy public key to VM using sshpass for initial auth. - // Retry because password auth may not be ready immediately after boot. - pubKey, err := os.ReadFile(keyPath + ".pub") - if err != nil { - t.Fatalf("failed to read public key: %v", err) - } - - const maxRetries = 12 - for attempt := 1; attempt <= maxRetries; attempt++ { - copyCmd := exec.Command("sshpass", "-p", vmPassword, - "ssh", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "PreferredAuthentications=password", - "-o", "PubkeyAuthentication=no", - "-o", "ConnectTimeout=10", - fmt.Sprintf("%s@%s", vm.User, vm.IP), - fmt.Sprintf("mkdir -p ~/.ssh && echo %q >> ~/.ssh/authorized_keys && chmod 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys", string(pubKey)), - ) - output, err := copyCmd.CombinedOutput() - if err == nil { - t.Logf("SSH key installed in VM (attempt %d)", attempt) - return - } - if attempt == maxRetries { - t.Fatalf("failed to copy SSH key to VM after %d attempts: %v, output: %s", maxRetries, err, string(output)) - } - t.Logf("SSH key copy attempt %d failed, retrying in 5s: %s", attempt, strings.TrimSpace(string(output))) - time.Sleep(5 * time.Second) - } -} - -func (vm *TartVM) waitForSSH(t *testing.T) { - t.Helper() - t.Logf("waiting for SSH readiness...") - - deadline := time.Now().Add(sshTimeout) - for time.Now().Before(deadline) { - args := vm.sshArgs() - args = append(args, "-o", "ConnectTimeout=5", vm.sshTarget(), "echo ok") - - cmd := exec.Command("ssh", args...) - output, err := cmd.CombinedOutput() - if err == nil && strings.TrimSpace(string(output)) == "ok" { - t.Logf("SSH is ready") - return - } - time.Sleep(sshPollInterval) - } - t.Fatalf("timed out waiting for SSH after %v", sshTimeout) -} - -func (vm *TartVM) sshArgs() []string { - keyPath := filepath.Join(vm.sshKeyDir, "id_ed25519") - return []string{ - "-i", keyPath, - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "LogLevel=ERROR", - } -} - -func (vm *TartVM) scpArgs() []string { - keyPath := filepath.Join(vm.sshKeyDir, "id_ed25519") - return []string{ - "-i", keyPath, - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "LogLevel=ERROR", - } -} - -func (vm *TartVM) sshTarget() string { - return fmt.Sprintf("%s@%s", vm.User, vm.IP) -} - -func requireTart(t *testing.T) { - t.Helper() - if _, err := exec.LookPath("tart"); err != nil { - t.Skip("tart not found; install with: brew install cirruslabs/cli/tart") - } -} - -func requireSSHPass(t *testing.T) { - t.Helper() - if _, err := exec.LookPath("sshpass"); err != nil { - t.Skip("sshpass not found; install with: brew install hudochenkov/sshpass/sshpass") - } -} - -func runCmd(t *testing.T, name string, args ...string) string { - t.Helper() - cmd := exec.Command(name, args...) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("%s %v failed: %v, output: %s", name, args, err, string(output)) - } - return string(output) -}