feat: add integration tests

.github/workflows/tests.yaml

@@ -0,0 +1,33 @@
+name: Tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  integration:
+    name: Integration (no machine)
+    # The provider binary is darwin-only, so the suite runs on macOS.
+    runs-on: macos-15
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          persist-credentials: false
+
+      - name: setup Go
+        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
+        with:
+          go-version-file: go.mod
+
+      - name: install goreleaser
+        uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7
+        with:
+          version: "~> v2"
+          args: release --snapshot --clean
+
+      - name: integration tests
+        working-directory: ./e2e
+        run: go test -tags=e2e -v -ginkgo.v -timeout 3600s --ginkgo.label-filter=integration

.gitignore

@@ -1,3 +1,4 @@
 /dist/
 *.out
 .DS_Store
+/e2e/bin/

Taskfile.yml

@@ -26,6 +26,18 @@ tasks:
     desc: Run unit tests
     cmd: go test ./...
 
+  test:e2e:
+    desc: Run no-machine e2e integration tests (macOS; no OrbStack required)
+    deps: ["build:cli"]
+    dir: e2e
+    cmd: go test -tags=e2e -v -ginkgo.v -timeout 3600s --ginkgo.label-filter=integration
+
+  test:e2e:vm:
+    desc: Run full machine-lifecycle e2e tests (requires OrbStack)
+    deps: ["build:cli"]
+    dir: e2e
+    cmd: go test -tags=e2e -v -ginkgo.v -timeout 3600s --ginkgo.label-filter=vm
+
   lint:
     desc: Run go vet
     cmd: go vet ./...

e2e/README.md

@@ -0,0 +1,23 @@
+## E2E tests
+
+The suite has two label groups (both require macOS, since the provider binary is
+darwin-only):
+
+- `integration` builds the provider and exercises the no-machine paths (manifest
+  generation, option validation, provider-binary error handling). It does not
+  require OrbStack to be installed or running.
+- `vm` provisions a real OrbStack machine and runs a full workspace lifecycle.
+  It requires OrbStack (`orbctl` on `PATH`, OrbStack running). This label does
+  not run in CI: OrbStack cannot start on GitHub-hosted macOS runners (they lack
+  the Apple Virtualization capability it needs). Run it on a real Mac or a
+  self-hosted macOS runner.
+
+### Run
+
+Build the provider binaries first, then run a label group from this directory:
+
+```sh
+task build:cli
+go test -v -ginkgo.v -timeout 3600s --ginkgo.label-filter=integration
+go test -v -ginkgo.v -timeout 3600s --ginkgo.label-filter=vm
+```

e2e/e2e_suite_test.go

@@ -0,0 +1,16 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"testing"
+
+	_ "github.com/devsy-org/devsy-provider-orbstack/e2e/tests/integration"
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+)
+
+func TestRunE2ETests(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "Devsy OrbStack Provider e2e suite")
+}

e2e/fixtures/workspace/.devcontainer/devcontainer.json

@@ -0,0 +1,4 @@
+{
+  "name": "orbstack-e2e",
+  "image": "mcr.microsoft.com/devcontainers/base:alpine"
+}

e2e/tests/integration/integration.go

@@ -0,0 +1,172 @@
+package integration
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	goruntime "runtime"
+	"strings"
+	"time"
+
+	"github.com/devsy-org/devsy/e2e/framework"
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+)
+
+// uniqueWorkspaceID returns a per-run workspace ID so the vm suite never
+// collides with or deletes a workspace a developer already has.
+func uniqueWorkspaceID() string {
+	b := make([]byte, 4)
+	_, err := rand.Read(b)
+	framework.ExpectNoError(err)
+	return "devsy-orbstack-e2e-" + hex.EncodeToString(b)
+}
+
+func providerBinary() string {
+	archDir := map[string]string{"amd64": "amd64_v1", "arm64": "arm64_v8.0"}[goruntime.GOARCH]
+	dir := fmt.Sprintf("build_%s_%s", goruntime.GOOS, archDir)
+	bin := fmt.Sprintf("devsy-provider-orbstack-%s-%s", goruntime.GOOS, goruntime.GOARCH)
+	return filepath.Join("..", "dist", dir, bin)
+}
+
+func mustRun(cmd *exec.Cmd) {
+	out, err := cmd.CombinedOutput()
+	gomega.Expect(err).NotTo(gomega.HaveOccurred(), string(out))
+}
+
+func isolateDevsyHome() {
+	dir, err := os.MkdirTemp("", "devsy-orbstack-e2e-")
+	framework.ExpectNoError(err)
+	framework.ExpectNoError(os.Setenv("DEVSY_HOME", dir))
+}
+
+func setupProvider() {
+	cmd := exec.Command("go", "run", "hack/provider/main.go", "0.0.0")
+	cmd.Dir = "../"
+	projectRoot, err := filepath.Abs("../")
+	framework.ExpectNoError(err)
+	cmd.Env = append(os.Environ(), "PROJECT_ROOT="+filepath.Join(projectRoot, "dist"))
+
+	output, err := cmd.Output()
+	framework.ExpectNoError(err)
+	framework.ExpectNoError(os.WriteFile("../dist/provider.yaml", output, 0o600))
+}
+
+func setupDevsyCLI() {
+	client := &http.Client{Timeout: 30 * time.Second}
+	url := fmt.Sprintf(
+		"https://github.com/devsy-org/devsy/releases/latest/download/devsy-%s-%s",
+		goruntime.GOOS, goruntime.GOARCH,
+	)
+	resp, err := client.Get(url) //nolint:gosec // fixed release URL
+	framework.ExpectNoError(err)
+	defer func() { _ = resp.Body.Close() }()
+	framework.ExpectEqual(resp.StatusCode, http.StatusOK)
+
+	framework.ExpectNoError(os.MkdirAll("bin", 0o750))
+	binPath := filepath.Join("bin", "devsy")
+	//nolint:gosec // fixed path; the devsy binary needs the executable bit
+	out, err := os.OpenFile(binPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755)
+	framework.ExpectNoError(err)
+	_, err = io.Copy(out, resp.Body)
+	framework.ExpectNoError(err)
+	// Close before exec: running a file still open for write triggers ETXTBSY.
+	framework.ExpectNoError(out.Close())
+	framework.ExpectNoError(exec.Command(binPath, "--version").Run()) //nolint:gosec // fixed path
+}
+
+var _ = ginkgo.Describe(
+	"devsy provider orbstack integration",
+	ginkgo.Label("integration"),
+	ginkgo.Ordered,
+	func() {
+		ginkgo.BeforeAll(func() {
+			setupProvider()
+		})
+
+		ginkgo.It("should generate a non-empty provider.yaml", func() {
+			data, err := os.ReadFile("../dist/provider.yaml")
+			framework.ExpectNoError(err)
+			framework.ExpectNotEqual(len(data), 0)
+		})
+
+		ginkgo.It("should have required provider options", func() {
+			data, err := os.ReadFile("../dist/provider.yaml")
+			framework.ExpectNoError(err)
+			content := string(data)
+			for _, opt := range []string{"ORBSTACK_DISTRO", "ORBSTACK_ISOLATED", "AGENT_PATH"} {
+				gomega.Expect(content).To(gomega.ContainSubstring(opt))
+			}
+		})
+
+		ginkgo.It("should fail init when orbctl is missing", func() {
+			cmd := exec.Command(providerBinary(), "init") //nolint:gosec // built provider path
+			cmd.Env = append(cmd.Environ(), "ORBSTACK_PATH=/nonexistent/orbctl")
+			output, err := cmd.CombinedOutput()
+			framework.ExpectError(err)
+			gomega.Expect(string(output)).To(gomega.ContainSubstring("orbctl not found"))
+		})
+
+		ginkgo.It("should fail create without a machine id", func() {
+			cmd := exec.Command(providerBinary(), "create") //nolint:gosec // built provider path
+			cmd.Env = append(cmd.Environ(), "MACHINE_ID=")
+			output, err := cmd.CombinedOutput()
+			framework.ExpectError(err)
+			gomega.Expect(string(output)).To(gomega.ContainSubstring("MACHINE_ID"))
+		})
+	},
+)
+
+var _ = ginkgo.Describe(
+	"devsy provider orbstack vm lifecycle",
+	ginkgo.Label("vm"),
+	ginkgo.Ordered,
+	func() {
+		var workspaceID string
+
+		ginkgo.BeforeAll(func() {
+			workspaceID = uniqueWorkspaceID()
+			isolateDevsyHome()
+			setupProvider()
+			setupDevsyCLI()
+			mustRun(exec.Command("bin/devsy", "provider", "add", "../dist/provider.yaml",
+				"--name", "orbstack", "-o", "ORBSTACK_CPUS=2", "-o", "ORBSTACK_MEMORY=4G"))
+
+			ginkgo.DeferCleanup(func() {
+				//nolint:gosec // fixed devsy binary; test-controlled args
+				_ = exec.Command("bin/devsy", "workspace", "delete", "--force", workspaceID).Run()
+			})
+		})
+
+		ginkgo.It("should bring up a workspace on an OrbStack machine", func() {
+			//nolint:gosec // fixed devsy binary; test-controlled args
+			mustRun(exec.Command("bin/devsy", "workspace", "up", "--ide=none",
+				"--id", workspaceID, "--provider", "orbstack", "fixtures/workspace"))
+		})
+
+		ginkgo.It("should run a command in the workspace", func() {
+			//nolint:gosec // fixed devsy binary; test-controlled args
+			cmd := exec.Command(
+				"bin/devsy",
+				"workspace",
+				"ssh",
+				workspaceID,
+				"--command",
+				"echo test",
+			)
+			output, err := cmd.Output()
+			framework.ExpectNoError(err)
+			gomega.Expect(strings.TrimSpace(string(output))).To(gomega.Equal("test"))
+		})
+
+		ginkgo.It("should delete the workspace", func() {
+			//nolint:gosec // fixed devsy binary; test-controlled args
+			mustRun(exec.Command("bin/devsy", "workspace", "delete", "--force", workspaceID))
+		})
+	},
+)