From 0e82771d698d0bf1f49877aeb316d7645f1036ce Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 26 Jul 2026 11:46:07 -0500 Subject: [PATCH 1/5] feat(driver): add microsandbox microVM provider Add a "microsandbox" backend that boots the devcontainer OCI image as a hardware-isolated microVM (libkrun) via the microsandbox runtime, wired through Devsy's driver/provider system. - New pkg/driver/microsandbox driver behind a narrow sandboxClient port (SDK isolated in one adapter; lifecycle logic unit tested with a fake). - Full up/exec/ssh/stop/restart/delete lifecycle; image-only devcontainers boot directly, built devcontainers build with docker then load into the runtime; authenticated registry pulls via go-containerregistry. - Named-volume/tmpfs mounts, idle-shutdown, resource ceilings, egress hardening, GPU host-requirement handling, cross-arch/name-length guards. - Offline agent delivery via DEVSY_AGENT_BINARY; provider.yaml + provider registration; desktop UI provider entry with a VM icon; e2e test and CI matrix entry (Linux/KVM, macOS Apple silicon). --- .github/workflows/pr-ci.yml | 15 + .../public/icons/providers/microsandbox.svg | 6 + .../components/provider/ProviderIcon.svelte | 2 + .../components/provider/ProviderWizard.svelte | 1 + e2e/tests/up/provider_microsandbox.go | 83 +++ .../testdata/microsandbox/.devcontainer.json | 4 + e2e/tests/up/up_behaviors.go | 4 +- pkg/agent/binary.go | 20 + pkg/agent/binary_env_test.go | 37 ++ pkg/agent/delivery/factory.go | 9 + pkg/agent/delivery/factory_test.go | 32 ++ pkg/config/env.go | 5 + pkg/driver/drivercreate/create.go | 8 +- pkg/driver/microsandbox/cliclient.go | 328 ++++++++++++ pkg/driver/microsandbox/client.go | 59 +++ pkg/driver/microsandbox/integration_test.go | 85 +++ pkg/driver/microsandbox/microsandbox.go | 479 +++++++++++++++++ pkg/driver/microsandbox/microsandbox_test.go | 501 ++++++++++++++++++ pkg/options/resolve.go | 29 + pkg/options/resolve_test.go | 26 + pkg/provider/parse.go | 7 +- pkg/provider/provider.go | 35 +- providers/microsandbox/provider.yaml | 49 ++ providers/providers.go | 14 +- providers/providers_test.go | 13 + 25 files changed, 1836 insertions(+), 15 deletions(-) create mode 100644 desktop/src/renderer/public/icons/providers/microsandbox.svg create mode 100644 e2e/tests/up/provider_microsandbox.go create mode 100644 e2e/tests/up/testdata/microsandbox/.devcontainer.json create mode 100644 pkg/agent/binary_env_test.go create mode 100644 pkg/driver/microsandbox/cliclient.go create mode 100644 pkg/driver/microsandbox/client.go create mode 100644 pkg/driver/microsandbox/integration_test.go create mode 100644 pkg/driver/microsandbox/microsandbox.go create mode 100644 pkg/driver/microsandbox/microsandbox_test.go create mode 100644 providers/microsandbox/provider.yaml diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 5d4698886..a07129c0c 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -405,6 +405,13 @@ jobs: install-kind: true requires-secret: false + - label: up-provider-microsandbox + runner: ubuntu-latest + free-disk-space: true + install-kind: false + requires-secret: false + install-microsandbox: true + # Up Docker Compose tests - label: up-docker-compose @@ -599,6 +606,14 @@ jobs: echo "DOCKER_HOST=unix:///run/podman/podman.sock" >> "$GITHUB_ENV" sudo podman info + - name: Install microsandbox (Linux) + if: matrix.install-microsandbox == true && runner.os == 'Linux' + run: | + curl -fsSL https://install.microsandbox.dev | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + # The e2e test skips gracefully if KVM is unavailable on the runner. + "$HOME/.local/bin/msb" doctor || true + - name: remove docker if: matrix.label == 'docker-install' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') run: | diff --git a/desktop/src/renderer/public/icons/providers/microsandbox.svg b/desktop/src/renderer/public/icons/providers/microsandbox.svg new file mode 100644 index 000000000..9a8d684af --- /dev/null +++ b/desktop/src/renderer/public/icons/providers/microsandbox.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/desktop/src/renderer/src/lib/components/provider/ProviderIcon.svelte b/desktop/src/renderer/src/lib/components/provider/ProviderIcon.svelte index 86abf2a28..e7e4b9d29 100644 --- a/desktop/src/renderer/src/lib/components/provider/ProviderIcon.svelte +++ b/desktop/src/renderer/src/lib/components/provider/ProviderIcon.svelte @@ -18,6 +18,8 @@ const ICON_MAP: Record = { kubernetes: "./icons/providers/kubernetes.svg", k8s: "./icons/providers/kubernetes.svg", k3s: "./icons/providers/kubernetes.svg", + microsandbox: "./icons/providers/microsandbox.svg", + microvm: "./icons/providers/microsandbox.svg", digitalocean: "./icons/providers/digitalocean.svg", ssh: "./icons/providers/ssh.svg", civo: "./icons/providers/civo.svg", diff --git a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte index 80c77b070..31e5d2b3c 100644 --- a/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte +++ b/desktop/src/renderer/src/lib/components/provider/ProviderWizard.svelte @@ -38,6 +38,7 @@ const PRESETS = [ { name: "docker", description: "Local Docker containers" }, { name: "podman", description: "Local Podman containers" }, { name: "apple", description: "Apple containers (macOS 26+, Apple silicon)" }, + { name: "microsandbox", description: "Hardware-isolated microVMs" }, { name: "lima", description: "Linux VMs via Lima (macOS)" }, { name: "orbstack", description: "Fast containers and VMs via OrbStack (macOS)" }, { name: "ssh", description: "Remote SSH machines" }, diff --git a/e2e/tests/up/provider_microsandbox.go b/e2e/tests/up/provider_microsandbox.go new file mode 100644 index 000000000..fa4d33389 --- /dev/null +++ b/e2e/tests/up/provider_microsandbox.go @@ -0,0 +1,83 @@ +package up + +import ( + "context" + "os" + "os/exec" + "runtime" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/onsi/ginkgo/v2" +) + +const osLinux = "linux" + +// skipIfNoMicrosandbox skips when the microsandbox runtime or hardware +// virtualization is unavailable, mirroring how other providers guard on their +// runtime being present. +func skipIfNoMicrosandbox() { + if _, err := exec.LookPath("msb"); err != nil { + ginkgo.Skip("microsandbox runtime (msb) not found on PATH") + } + switch { + case runtime.GOOS == osLinux: + kvm, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) + if err != nil { + ginkgo.Skip("microsandbox requires KVM (/dev/kvm not accessible)") + } + _ = kvm.Close() + case runtime.GOOS == "darwin" && runtime.GOARCH == "arm64": + // Apple silicon supports microsandbox via the hypervisor framework. + default: + ginkgo.Skip("microsandbox requires Apple silicon or Linux with KVM") + } +} + +var _ = ginkgo.Describe( + "testing up command for microsandbox provider", + ginkgo.Label("up-provider-microsandbox"), + func() { + var initialDir string + + ginkgo.BeforeEach(func() { + skipIfNoMicrosandbox() + var err error + initialDir, err = os.Getwd() + framework.ExpectNoError(err) + }) + + ginkgo.It("runs devsy in a microsandbox microVM", func(ctx context.Context) { + f := framework.NewDefaultFramework(initialDir + "/bin") + tempDir, err := framework.CopyToTempDir("tests/up/testdata/microsandbox") + framework.ExpectNoError(err) + ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, tempDir) + + _ = f.DevsyProviderDelete(ctx, "microsandbox") + err = f.DevsyProviderAdd(ctx, "microsandbox") + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + err := f.DevsyProviderDelete(cleanupCtx, "microsandbox") + framework.ExpectNoError(err) + }) + + // full up: boots the microVM, streams in the agent, opens the tunnel + err = f.DevsyUp(ctx, tempDir) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(f.DevsyWorkspaceDelete, tempDir) + + // the workspace is reachable over SSH + err = f.DevsySSHEchoTestString(ctx, tempDir) + framework.ExpectNoError(err) + + // stop, then bring it back up and confirm it is reachable again + err = f.DevsyWorkspaceStop(ctx, tempDir) + framework.ExpectNoError(err) + + err = f.DevsyUp(ctx, tempDir) + framework.ExpectNoError(err) + + err = f.DevsySSHEchoTestString(ctx, tempDir) + framework.ExpectNoError(err) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) + }, +) diff --git a/e2e/tests/up/testdata/microsandbox/.devcontainer.json b/e2e/tests/up/testdata/microsandbox/.devcontainer.json new file mode 100644 index 000000000..ca9e68883 --- /dev/null +++ b/e2e/tests/up/testdata/microsandbox/.devcontainer.json @@ -0,0 +1,4 @@ +{ + "name": "microsandbox", + "image": "mcr.microsoft.com/devcontainers/base:ubuntu" +} diff --git a/e2e/tests/up/up_behaviors.go b/e2e/tests/up/up_behaviors.go index 8ad62e2a9..de58bc4fb 100644 --- a/e2e/tests/up/up_behaviors.go +++ b/e2e/tests/up/up_behaviors.go @@ -118,7 +118,7 @@ var _ = ginkgo.Describe("up command behaviors", ginkgo.Label("up-behaviors"), fu ) ginkgo.It("should accept --update-remote-user-uid=on", func(ctx context.Context) { - if runtime.GOOS != "linux" { + if runtime.GOOS != osLinux { ginkgo.Skip("updateRemoteUserUID only applies on Linux") } _, err := dtc.setupAndUp(ctx, "tests/up/testdata/docker", @@ -127,7 +127,7 @@ var _ = ginkgo.Describe("up command behaviors", ginkgo.Label("up-behaviors"), fu }, ginkgo.SpecTimeout(framework.TimeoutShort())) ginkgo.It("should accept --update-remote-user-uid=off", func(ctx context.Context) { - if runtime.GOOS != "linux" { + if runtime.GOOS != osLinux { ginkgo.Skip("updateRemoteUserUID only applies on Linux") } tempDir, err := dtc.setupAndUp(ctx, "tests/up/testdata/docker", diff --git a/pkg/agent/binary.go b/pkg/agent/binary.go index b1e01ba6b..bac72614f 100644 --- a/pkg/agent/binary.go +++ b/pkg/agent/binary.go @@ -37,6 +37,7 @@ func NewBinaryManager(downloadURL string) (*BinaryManager, error) { return &BinaryManager{ sources: []BinarySource{ + &EnvPathSource{EnvVar: config.EnvAgentBinary}, &InjectSource{}, &FileCacheSource{Cache: cache, ExpectedVersion: expectedVersion}, &HTTPDownloadSource{BaseURL: downloadURL, Cache: cache, Version: expectedVersion}, @@ -44,6 +45,25 @@ func NewBinaryManager(downloadURL string) (*BinaryManager, error) { }, nil } +// EnvPathSource injects an agent binary from a local path named by an +// environment variable, taking precedence over the built-in sources so an +// offline host can supply the Linux binary. The operator must match its arch. +type EnvPathSource struct { + EnvVar string +} + +func (s *EnvPathSource) GetBinary(_ context.Context, _ string) (io.ReadCloser, error) { + path := strings.TrimSpace(os.Getenv(s.EnvVar)) + if path == "" { + return nil, fmt.Errorf("no agent binary path set in %s", s.EnvVar) + } + return os.Open(path) // #nosec G304 G703 -- operator-provided local agent binary +} + +func (s *EnvPathSource) SourceName() string { + return "local path override (" + s.EnvVar + ")" +} + func versionFromDownloadURL(downloadURL string) string { parts := strings.Split(strings.TrimRight(downloadURL, "/"), "/") if len(parts) == 0 { diff --git a/pkg/agent/binary_env_test.go b/pkg/agent/binary_env_test.go new file mode 100644 index 000000000..cf6a9a45e --- /dev/null +++ b/pkg/agent/binary_env_test.go @@ -0,0 +1,37 @@ +package agent + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" +) + +func TestEnvPathSourceUsesLocalBinary(t *testing.T) { + dir := t.TempDir() + binPath := filepath.Join(dir, "devsy-linux-arm64") + if err := os.WriteFile(binPath, []byte("agent-bytes"), 0o755); err != nil { //nolint:gosec + t.Fatal(err) + } + t.Setenv("DEVSY_TEST_AGENT_BIN", binPath) + + src := &EnvPathSource{EnvVar: "DEVSY_TEST_AGENT_BIN"} + rc, err := src.GetBinary(context.Background(), "arm64") + if err != nil { + t.Fatalf("GetBinary: %v", err) + } + defer func() { _ = rc.Close() }() + + data, _ := io.ReadAll(rc) + if string(data) != "agent-bytes" { + t.Errorf("got %q, want the local binary contents", data) + } +} + +func TestEnvPathSourceUnsetFallsThrough(t *testing.T) { + src := &EnvPathSource{EnvVar: "DEVSY_TEST_AGENT_BIN_UNSET"} + if _, err := src.GetBinary(context.Background(), "arm64"); err == nil { + t.Fatal("expected an error when the env var is unset so later sources are tried") + } +} diff --git a/pkg/agent/delivery/factory.go b/pkg/agent/delivery/factory.go index 7701c0a42..efdf0a23d 100644 --- a/pkg/agent/delivery/factory.go +++ b/pkg/agent/delivery/factory.go @@ -38,6 +38,15 @@ func NewAgentDelivery(opts FactoryOptions) AgentDelivery { log.Debugf("using shell-based delivery for apple driver") return &LegacyShellDelivery{ExecFunc: opts.ExecFunc, DownloadURL: ""} + case driverType == provider.MicrosandboxDriver: + // Stream the agent binary over the SDK's guest exec (as kubernetes does); + // fall back to shell delivery when the driver exposes no argv exec. + if opts.PodExec == nil { + return legacyShellDelivery(opts, "microsandbox argv exec unavailable") + } + log.Debugf("using stream delivery (exec stream) for microsandbox") + return &KubernetesDelivery{Exec: opts.PodExec} + case opts.IsRemoteDocker: log.Debugf("using remote docker delivery (docker cp)") return remoteDockerDelivery(opts) diff --git a/pkg/agent/delivery/factory_test.go b/pkg/agent/delivery/factory_test.go index a2672ac26..5d23ed38c 100644 --- a/pkg/agent/delivery/factory_test.go +++ b/pkg/agent/delivery/factory_test.go @@ -95,6 +95,26 @@ func TestNewAgentDelivery_KubernetesDriver_Native(t *testing.T) { assert.Equal(t, PhasePostStart, d.Phase()) } +func TestNewAgentDelivery_MicrosandboxUsesStreamDelivery(t *testing.T) { + podExec := func(_ context.Context, _ []string, _ driver.Streams) error { + return nil + } + + opts := FactoryOptions{ + WorkspaceConfig: &provider.AgentWorkspaceInfo{ + Agent: provider.ProviderAgentConfig{ + Driver: provider.MicrosandboxDriver, + }, + }, + PodExec: podExec, + } + + d := NewAgentDelivery(opts) + native, ok := d.(*KubernetesDelivery) + require.True(t, ok) + assert.NotNil(t, native.Exec) +} + func TestNewAgentDelivery_KubernetesDriver_FallsBackWhenNoPodExec(t *testing.T) { execFn := func(ctx context.Context, cmd string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return nil @@ -168,3 +188,15 @@ func TestNewAgentDelivery_AppleUsesShellDelivery(t *testing.T) { t.Fatalf("apple driver must use shell delivery, got %T", d) } } + +func TestNewAgentDelivery_MicrosandboxUsesShellDelivery(t *testing.T) { + opts := FactoryOptions{ + WorkspaceConfig: &provider.AgentWorkspaceInfo{ + Agent: provider.ProviderAgentConfig{Driver: provider.MicrosandboxDriver}, + }, + } + d := NewAgentDelivery(opts) + if _, ok := d.(*LegacyShellDelivery); !ok { + t.Fatalf("microsandbox driver must use shell delivery, got %T", d) + } +} diff --git a/pkg/config/env.go b/pkg/config/env.go index 29990484d..410f0e5d5 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -45,6 +45,11 @@ const ( // EnvHelperImage overrides the helper image for agent volume operations. EnvHelperImage = "DEVSY_HELPER_IMAGE" + // EnvAgentBinary points to a local agent binary (matching the workspace's + // Linux architecture) to inject instead of downloading. Enables offline and + // air-gapped delivery on hosts that cannot supply the binary directly. + EnvAgentBinary = "DEVSY_AGENT_BINARY" + // EnvAgentPreferDownload forces agent binary download even if a local copy exists. EnvAgentPreferDownload = "DEVSY_AGENT_PREFER_DOWNLOAD" diff --git a/pkg/driver/drivercreate/create.go b/pkg/driver/drivercreate/create.go index e690824db..e085bca9d 100644 --- a/pkg/driver/drivercreate/create.go +++ b/pkg/driver/drivercreate/create.go @@ -9,6 +9,7 @@ import ( "github.com/devsy-org/devsy/pkg/driver/custom" "github.com/devsy-org/devsy/pkg/driver/docker" "github.com/devsy-org/devsy/pkg/driver/kubernetes" + "github.com/devsy-org/devsy/pkg/driver/microsandbox" provider2 "github.com/devsy-org/devsy/pkg/provider" ) @@ -26,9 +27,12 @@ func NewDriver( return kubernetes.NewKubernetesDriver(workspaceInfo) case provider2.AppleDriver: return apple.NewAppleDriver(ctx, workspaceInfo) + case provider2.MicrosandboxDriver: + return microsandbox.NewMicrosandboxDriver(ctx, workspaceInfo) } - return nil, fmt.Errorf("unrecognized driver %q, possible values are %s, %s, %s or %s", + return nil, fmt.Errorf( + "unrecognized driver %q, possible values are %s, %s, %s, %s or %s", driver, provider2.DockerDriver, provider2.CustomDriver, provider2.KubernetesDriver, - provider2.AppleDriver) + provider2.AppleDriver, provider2.MicrosandboxDriver) } diff --git a/pkg/driver/microsandbox/cliclient.go b/pkg/driver/microsandbox/cliclient.go new file mode 100644 index 000000000..894ee20d3 --- /dev/null +++ b/pkg/driver/microsandbox/cliclient.go @@ -0,0 +1,328 @@ +package microsandbox + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/devsy-org/devsy/pkg/image" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +type cliClient struct{} + +var _ sandboxClient = cliClient{} + +func (cliClient) EnsureInstalled(_ context.Context) error { + bin := msbBinary() + if filepath.IsAbs(bin) { + if _, err := os.Stat(bin); err == nil { + return nil + } + } else if _, err := exec.LookPath(bin); err == nil { + return nil + } + return fmt.Errorf( + "microsandbox runtime (msb) not found; install it from https://install.microsandbox.dev", + ) +} + +func (cliClient) EnsureImage(ctx context.Context, imageRef string) error { + if dockerImageExists(ctx, imageRef) { + return loadFromDocker(ctx, imageRef) + } + // #nosec G204 -- args are a resolved binary path and a validated image ref + out, err := exec.CommandContext(ctx, msbBinary(), "pull", imageRef).CombinedOutput() + if err == nil { + return nil + } + if loadErr := loadViaRegistry(ctx, imageRef); loadErr != nil { + return fmt.Errorf("msb pull %s: %s: %w; registry fallback: %v", imageRef, out, err, loadErr) + } + return nil +} + +func (c cliClient) Create(ctx context.Context, sandbox string, spec sandboxSpec) error { + if err := c.ensureVolumes(ctx, spec.Mounts); err != nil { + return err + } + return msbRun(ctx, createArgs(sandbox, spec)...) +} + +func (cliClient) Find(ctx context.Context, sandbox string) (*sandboxInfo, error) { + // #nosec G204 -- args are a resolved binary path and a derived sandbox name + out, err := exec.CommandContext(ctx, msbBinary(), "inspect", sandbox, "--format", "json"). + Output() + if err != nil { + // inspect fails when the sandbox is absent, which is not an error here. + // A cancelled/timed-out context is, so do not mask it as "not found". + if ctx.Err() != nil { + return nil, fmt.Errorf("inspect microsandbox VM %q: %w", sandbox, ctx.Err()) + } + return nil, nil + } + var raw struct { + Name string `json:"name"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + ActiveConfig struct { + Labels map[string]string `json:"labels"` + } `json:"active_config"` + } + if err := json.Unmarshal(out, &raw); err != nil { + return nil, fmt.Errorf("parse microsandbox inspect output: %w", err) + } + created, _ := time.Parse(time.RFC3339Nano, raw.CreatedAt) + return &sandboxInfo{ + Name: raw.Name, + Running: strings.EqualFold(raw.Status, "running"), + CreatedAt: created, + Labels: raw.ActiveConfig.Labels, + }, nil +} + +func (cliClient) Start(ctx context.Context, sandbox string) error { + return msbRun(ctx, "start", sandbox) +} + +func (cliClient) Stop(ctx context.Context, sandbox string) error { + return msbRun(ctx, "stop", sandbox) +} + +func (cliClient) Remove(ctx context.Context, sandbox string) error { + return msbRun(ctx, "remove", sandbox) +} + +// Exec uses --stream (byte-faithful stdio, no PTY) because the agent binary +// injection and the SSH-over-stdio tunnel require it; a PTY stalls the tunnel. +func (cliClient) Exec(ctx context.Context, sandbox string, req execRequest) error { + args := []string{"exec", "--stream"} + if req.User != "" { + args = append(args, "--user", req.User) + } + args = append(args, sandbox, "--") + if len(req.Argv) > 0 { + args = append(args, req.Argv...) + } else { + args = append(args, "/bin/sh", "-c", req.Command) + } + // #nosec G204 -- args are a resolved binary path plus the caller's command + cmd := exec.CommandContext(ctx, msbBinary(), args...) + cmd.Stdin = req.Stdin + cmd.Stdout = req.Stdout + cmd.Stderr = req.Stderr + return cmd.Run() +} + +func (cliClient) Logs(ctx context.Context, sandbox string, w io.Writer) error { + // #nosec G204 -- args are a resolved binary path and a derived sandbox name + cmd := exec.CommandContext(ctx, msbBinary(), "logs", sandbox) + cmd.Stdout = w + cmd.Stderr = w + return cmd.Run() +} + +func (cliClient) ensureVolumes(ctx context.Context, mounts []volumeMount) error { + for _, vol := range namedVolumes(mounts) { + // #nosec G204 -- args are a resolved binary path and a named-volume name + out, err := exec.CommandContext(ctx, msbBinary(), "volume", "create", vol).CombinedOutput() + if err != nil && !strings.Contains(strings.ToLower(string(out)), "exist") { + return fmt.Errorf("create microsandbox volume %q: %s: %w", vol, out, err) + } + } + return nil +} + +func createArgs(sandbox string, spec sandboxSpec) []string { + args := []string{"create", "--name", sandbox} + args = append(args, runtimeArgs(spec)...) + args = append(args, resourceArgs(spec)...) + args = append(args, mountArgs(spec.Mounts)...) + return append(args, spec.Image) +} + +func runtimeArgs(spec sandboxSpec) []string { + var args []string + if len(spec.Entrypoint) > 0 { + args = append(args, "--entrypoint", strings.Join(spec.Entrypoint, " ")) + } + for k, v := range spec.Env { + args = append(args, "--env", k+"="+v) + } + for k, v := range spec.Labels { + args = append(args, "--label", k+"="+v) + } + if spec.IdleTimeout > 0 { + args = append(args, "--idle-timeout", spec.IdleTimeout.String()) + } + if spec.BlockEgress { + args = append(args, "--net-default-egress", "deny") + } + return args +} + +func resourceArgs(spec sandboxSpec) []string { + var args []string + if spec.Memory > 0 { + args = append(args, "--memory", fmt.Sprintf("%dM", spec.Memory)) + } + if spec.MaxMemory > 0 { + args = append(args, "--max-memory", fmt.Sprintf("%dM", spec.MaxMemory)) + } + if spec.CPUs > 0 { + args = append(args, "--cpus", strconv.Itoa(int(spec.CPUs))) + } + if spec.MaxCPUs > 0 { + args = append(args, "--max-cpus", strconv.Itoa(int(spec.MaxCPUs))) + } + return args +} + +func mountArgs(mounts []volumeMount) []string { + var args []string + for _, m := range mounts { + switch { + case m.Tmpfs: + args = append(args, "--tmpfs", m.Target) + case m.Volume != "": + args = append(args, "--mount-named", m.Volume+":"+m.Target) + } + } + return args +} + +func namedVolumes(mounts []volumeMount) []string { + var names []string + for _, m := range mounts { + if !m.Tmpfs && m.Volume != "" { + names = append(names, m.Volume) + } + } + return names +} + +func msbRun(ctx context.Context, args ...string) error { + // #nosec G204 -- args are a resolved binary path and controlled config values + out, err := exec.CommandContext(ctx, msbBinary(), args...).CombinedOutput() + if err != nil { + return fmt.Errorf( + "msb %s: %s: %w", + redactArgs(args), + strings.TrimSpace(string(out)), + err, + ) + } + return nil +} + +// redactArgs joins args for an error message while masking --env values, which +// may carry secrets from the devcontainer configuration. +func redactArgs(args []string) string { + out := make([]string, len(args)) + copy(out, args) + for i := 0; i+1 < len(out); i++ { + if out[i] != "--env" { + continue + } + if k, _, ok := strings.Cut(out[i+1], "="); ok { + out[i+1] = k + "=***" + } + } + return strings.Join(out, " ") +} + +func dockerImageExists(ctx context.Context, image string) bool { + docker, err := exec.LookPath("docker") + if err != nil { + return false + } + // #nosec G204 -- docker path is resolved and the image ref is validated + return exec.CommandContext(ctx, docker, "image", "inspect", image).Run() == nil +} + +func loadFromDocker(ctx context.Context, image string) error { + docker, err := exec.LookPath("docker") + if err != nil { + return fmt.Errorf("docker not found to load built image %q: %w", image, err) + } + // #nosec G204 -- docker/msb paths are resolved and the image ref is validated + save := exec.CommandContext(ctx, docker, "save", image) + // #nosec G204 -- docker/msb paths are resolved and the image ref is validated + load := exec.CommandContext(ctx, msbBinary(), "load", "-t", image) + pipe, err := save.StdoutPipe() + if err != nil { + return fmt.Errorf("pipe docker save: %w", err) + } + load.Stdin = pipe + var loadErr strings.Builder + load.Stderr = &loadErr + if err := load.Start(); err != nil { + return fmt.Errorf("start msb load: %w", err) + } + if err := save.Run(); err != nil { + // load is still running on the broken pipe; kill and reap it. + _ = load.Process.Kill() + _ = load.Wait() + return fmt.Errorf("docker save %q: %w", image, err) + } + if err := load.Wait(); err != nil { + return fmt.Errorf("msb load %q: %s: %w", image, loadErr.String(), err) + } + return nil +} + +func loadViaRegistry(ctx context.Context, imageRef string) error { + ref, err := name.ParseReference(imageRef) + if err != nil { + return fmt.Errorf("parse image reference %q: %w", imageRef, err) + } + img, err := image.GetImageForArch(ctx, imageRef, runtime.GOARCH) + if err != nil { + return fmt.Errorf("pull image %q: %w", imageRef, err) + } + + tmp, err := os.CreateTemp("", "devsy-msb-*.tar") + if err != nil { + return fmt.Errorf("create image tarball: %w", err) + } + tmpPath := tmp.Name() + _ = tmp.Close() + defer func() { _ = os.Remove(tmpPath) }() + + if err := tarball.WriteToFile(tmpPath, ref, img); err != nil { + return fmt.Errorf("write image tarball: %w", err) + } + + // #nosec G204 -- args are a resolved binary path and an internally-created file + load := exec.CommandContext(ctx, msbBinary(), "load", "-i", tmpPath, "-t", imageRef) + if out, err := load.CombinedOutput(); err != nil { + return fmt.Errorf("msb load %q: %s: %w", imageRef, out, err) + } + return nil +} + +func msbBinary() string { + if p, err := exec.LookPath("msb"); err == nil { + return p + } + if home, err := os.UserHomeDir(); err == nil { + for _, c := range []string{ + filepath.Join(home, ".local", "bin", "msb"), + filepath.Join(home, ".microsandbox", "bin", "msb"), + } { + if _, statErr := os.Stat(c); statErr == nil { + return c + } + } + } + return "msb" +} diff --git a/pkg/driver/microsandbox/client.go b/pkg/driver/microsandbox/client.go new file mode 100644 index 000000000..aa2000cbc --- /dev/null +++ b/pkg/driver/microsandbox/client.go @@ -0,0 +1,59 @@ +package microsandbox + +import ( + "context" + "io" + "time" +) + +type sandboxSpec struct { + Image string + Entrypoint []string + Memory uint32 + CPUs uint8 + Env map[string]string + Labels map[string]string + Ephemeral bool + IdleTimeout time.Duration + Mounts []volumeMount + MaxMemory uint32 + MaxCPUs uint8 + BlockEgress bool +} + +type volumeMount struct { + Target string + Volume string + Tmpfs bool +} + +type sandboxInfo struct { + Name string + Running bool + CreatedAt time.Time + Labels map[string]string +} + +type execRequest struct { + Command string + Argv []string + User string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// sandboxClient abstracts the microsandbox runtime so the driver can be tested +// against a fake. Lifecycle ordering (e.g. stop before remove) is the driver's +// job, not the client's. +type sandboxClient interface { + EnsureInstalled(ctx context.Context) error + EnsureImage(ctx context.Context, image string) error + Create(ctx context.Context, name string, spec sandboxSpec) error + Find(ctx context.Context, name string) (*sandboxInfo, error) + Start(ctx context.Context, name string) error + Stop(ctx context.Context, name string) error + Remove(ctx context.Context, name string) error + Exec(ctx context.Context, name string, req execRequest) error + Logs(ctx context.Context, name string, w io.Writer) error +} diff --git a/pkg/driver/microsandbox/integration_test.go b/pkg/driver/microsandbox/integration_test.go new file mode 100644 index 000000000..08203f3ce --- /dev/null +++ b/pkg/driver/microsandbox/integration_test.go @@ -0,0 +1,85 @@ +//go:build microsandbox_integration + +// Integration test that drives the microsandbox driver against the real runtime. +// Requires the microsandbox runtime installed and network access to pull the +// image. Run with: go test -tags microsandbox_integration ./pkg/driver/microsandbox/ +package microsandbox + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/driver" +) + +func TestDriverLifecycleIntegration(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + d := newDriver(cliClient{}, nil, specDefaults{memory: 2048, cpus: 2}) + const workspaceID = "integration-test" + + // Clean up any residue from a prior run, and always at the end. + _ = d.DeleteDevContainer(ctx, workspaceID) + defer func() { _ = d.DeleteDevContainer(ctx, workspaceID) }() + + if err := d.RunDevContainer(ctx, workspaceID, &driver.RunOptions{ + Image: "mcr.microsoft.com/devcontainers/base:ubuntu", + }); err != nil { + t.Fatalf("RunDevContainer: %v", err) + } + + details, err := d.FindDevContainer(ctx, workspaceID) + if err != nil { + t.Fatalf("FindDevContainer: %v", err) + } + if details == nil { + t.Fatal("FindDevContainer returned nil for a running workspace") + } + if details.State.Status != "running" { + t.Errorf("status = %q, want running", details.State.Status) + } + + var stdout, stderr bytes.Buffer + if err := d.CommandDevContainer(ctx, &driver.CommandParams{ + WorkspaceID: workspaceID, + Command: "echo hello-from-vm && uname -r", + Stdout: &stdout, + Stderr: &stderr, + }); err != nil { + t.Fatalf("CommandDevContainer: %v (stderr=%s)", err, stderr.String()) + } + if !strings.Contains(stdout.String(), "hello-from-vm") { + t.Errorf("command stdout = %q, want it to contain hello-from-vm", stdout.String()) + } + t.Logf("command output:\n%s", stdout.String()) +} + +func TestLoadViaRegistryIntegration(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + const img = "mcr.microsoft.com/devcontainers/base:ubuntu" + if err := loadViaRegistry(ctx, img); err != nil { + t.Fatalf("loadViaRegistry (go-containerregistry pull -> tarball -> msb load): %v", err) + } + // Boot the loaded image and confirm it runs. + d := newDriver(cliClient{}, nil, specDefaults{}) + const workspaceID = "gcrload-test" + defer func() { _ = d.DeleteDevContainer(ctx, workspaceID) }() + if err := d.client.Create(ctx, sandboxName(workspaceID), sandboxSpec{Image: img}); err != nil { + t.Fatalf("create from registry-loaded image: %v", err) + } + var out bytes.Buffer + if err := d.CommandDevContainer(ctx, &driver.CommandParams{ + WorkspaceID: workspaceID, Command: "echo gcr-load-ok", Stdout: &out, + }); err != nil { + t.Fatalf("exec: %v", err) + } + if !strings.Contains(out.String(), "gcr-load-ok") { + t.Errorf("output = %q", out.String()) + } +} diff --git a/pkg/driver/microsandbox/microsandbox.go b/pkg/driver/microsandbox/microsandbox.go new file mode 100644 index 000000000..ba254b717 --- /dev/null +++ b/pkg/driver/microsandbox/microsandbox.go @@ -0,0 +1,479 @@ +// Package microsandbox is a Devsy driver that runs devcontainers as microVMs. +package microsandbox + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "runtime" + "strconv" + "strings" + "time" + + pkgconfig "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" + dockerdriver "github.com/devsy-org/devsy/pkg/driver/docker" + "github.com/devsy-org/devsy/pkg/image" + "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/provider" +) + +const userLabel = "devsy.sh/user" + +const createdAtLayout = "2006-01-02T15:04:05Z07:00" + +type specDefaults struct { + memory uint32 + cpus uint8 + ephemeral bool + idleTimeout time.Duration + maxMemory uint32 + maxCPUs uint8 + blockEgress bool +} + +type microsandboxDriver struct { + client sandboxClient + idLabels []string + defaults specDefaults + workspaceInfo *provider.AgentWorkspaceInfo + docker driver.ImageDriver +} + +var ( + _ driver.RunOptionsDriver = (*microsandboxDriver)(nil) + _ driver.ReprovisioningDriver = (*microsandboxDriver)(nil) + _ driver.ImageDriver = (*microsandboxDriver)(nil) +) + +// CanReprovision returns false: in-place reprovision passes nil options, which +// a microVM cannot rebuild from. --recreate still works via delete+create. +func (d *microsandboxDriver) CanReprovision() bool { + return false +} + +func NewMicrosandboxDriver( + ctx context.Context, + workspaceInfo *provider.AgentWorkspaceInfo, +) (driver.Driver, error) { + client := cliClient{} + if err := client.EnsureInstalled(ctx); err != nil { + return nil, fmt.Errorf("microsandbox runtime is not available: %w", err) + } + + cfg := workspaceInfo.Agent.Microsandbox + defaults := specDefaults{ + memory: parseUint32(cfg.Memory), + cpus: parseUint8(cfg.CPUs), + ephemeral: cfg.Ephemeral == pkgconfig.BoolTrue, + idleTimeout: parseDuration(workspaceInfo.Agent.ContainerTimeout), + maxMemory: parseUint32(cfg.MaxMemory), + maxCPUs: parseUint8(cfg.MaxCPUs), + blockEgress: cfg.BlockEgress == pkgconfig.BoolTrue, + } + + log.Debugf( + "using microsandbox driver: memory=%dMiB cpus=%d ephemeral=%t idleTimeout=%s", + defaults.memory, defaults.cpus, defaults.ephemeral, defaults.idleTimeout, + ) + d := newDriver(client, workspaceInfo.CLIOptions.IDLabels, defaults) + d.workspaceInfo = workspaceInfo + return d, nil +} + +func newDriver(client sandboxClient, idLabels []string, defaults specDefaults) *microsandboxDriver { + return µsandboxDriver{client: client, idLabels: idLabels, defaults: defaults} +} + +func (d *microsandboxDriver) RunDevContainer( + ctx context.Context, + workspaceID string, + options *driver.RunOptions, +) error { + return d.runFromOptions(ctx, workspaceID, options) +} + +func (d *microsandboxDriver) RunImageDevContainer( + ctx context.Context, + params *driver.RunImageDevContainerParams, +) error { + if err := checkGPURequirement(params.ParsedConfig); err != nil { + return err + } + return d.runFromOptions(ctx, params.WorkspaceID, params.Options) +} + +func checkGPURequirement(parsedConfig *config.DevContainerConfig) error { + if parsedConfig == nil || + parsedConfig.HostRequirements == nil || + parsedConfig.HostRequirements.GPU == nil { + return nil + } + switch parsedConfig.HostRequirements.GPU.Value { + case "true": + return fmt.Errorf( + "microsandbox does not support GPU passthrough, but hostRequirements.gpu is required; " + + "use a GPU-capable provider or set gpu to \"optional\"", + ) + case "optional": + log.Warnf( + "hostRequirements.gpu is optional; microsandbox provides no GPU, continuing without one", + ) + } + return nil +} + +func (d *microsandboxDriver) FindDevContainer( + ctx context.Context, + workspaceID string, +) (*config.ContainerDetails, error) { + info, err := d.client.Find(ctx, sandboxName(workspaceID)) + if err != nil || info == nil { + return nil, err + } + return toContainerDetails(info), nil +} + +func (d *microsandboxDriver) CommandDevContainer( + ctx context.Context, + params *driver.CommandParams, +) error { + return d.client.Exec(ctx, sandboxName(params.WorkspaceID), execRequest{ + Command: params.Command, + User: params.User, + Stdin: params.Stdin, + Stdout: params.Stdout, + Stderr: params.Stderr, + }) +} + +// CommandContainerArgv satisfies the podExecCapableDriver interface that +// pkg/devcontainer detects to stream the agent binary in over stdin. +func (d *microsandboxDriver) CommandContainerArgv( + ctx context.Context, + workspaceID string, + argv []string, + streams driver.Streams, +) error { + return d.client.Exec(ctx, sandboxName(workspaceID), execRequest{ + Argv: argv, + User: "root", + Stdin: streams.Stdin, + Stdout: streams.Stdout, + Stderr: streams.Stderr, + }) +} + +func (d *microsandboxDriver) TargetArchitecture( + ctx context.Context, + workspaceID string, +) (string, error) { + switch runtime.GOARCH { + case "amd64", "arm64": + return runtime.GOARCH, nil + default: + return "", fmt.Errorf("unsupported architecture %q for microsandbox", runtime.GOARCH) + } +} + +func (d *microsandboxDriver) InspectImage( + ctx context.Context, + imageName string, +) (*config.ImageDetails, error) { + arch, err := d.TargetArchitecture(ctx, "") + if err != nil { + return nil, err + } + cfg, _, err := image.GetImageConfigForArch(ctx, imageName, arch) + if err != nil { + return nil, fmt.Errorf("get image config for %s: %w", imageName, err) + } + return &config.ImageDetails{ + ID: imageName, + Config: config.ImageDetailsConfig{ + User: cfg.Config.User, + Env: cfg.Config.Env, + Labels: cfg.Config.Labels, + Entrypoint: cfg.Config.Entrypoint, + Cmd: cfg.Config.Cmd, + }, + }, nil +} + +func (d *microsandboxDriver) GetImageTag(_ context.Context, imageName string) (string, error) { + return imageName, nil +} + +func (d *microsandboxDriver) BuildDevContainer( + ctx context.Context, + req driver.BuildRequest, +) (*config.BuildInfo, error) { + dockerDriver, err := d.dockerImageDriver() + if err != nil { + return nil, err + } + return dockerDriver.BuildDevContainer(ctx, req) +} + +func (d *microsandboxDriver) PushDevContainer(ctx context.Context, image string) error { + dockerDriver, err := d.dockerImageDriver() + if err != nil { + return err + } + return dockerDriver.PushDevContainer(ctx, image) +} + +func (d *microsandboxDriver) TagDevContainer(ctx context.Context, image, tag string) error { + dockerDriver, err := d.dockerImageDriver() + if err != nil { + return err + } + return dockerDriver.TagDevContainer(ctx, image, tag) +} + +func (d *microsandboxDriver) UpdateContainerUserUID( + _ context.Context, + _ string, + _ *config.DevContainerConfig, + _ io.Writer, +) error { + return nil +} + +func (d *microsandboxDriver) StartDevContainer(ctx context.Context, workspaceID string) error { + if err := d.client.Start(ctx, sandboxName(workspaceID)); err != nil { + return fmt.Errorf("start microsandbox VM: %w", err) + } + return nil +} + +func (d *microsandboxDriver) StopDevContainer(ctx context.Context, workspaceID string) error { + if err := d.client.Stop(ctx, sandboxName(workspaceID)); err != nil { + return fmt.Errorf("stop microsandbox VM: %w", err) + } + return nil +} + +// DeleteDevContainer stops before removing; the runtime only removes stopped +// sandboxes. +func (d *microsandboxDriver) DeleteDevContainer(ctx context.Context, workspaceID string) error { + name := sandboxName(workspaceID) + info, err := d.client.Find(ctx, name) + if err != nil { + return err + } + if info == nil { + return nil + } + if info.Running { + if err := d.client.Stop(ctx, name); err != nil { + return fmt.Errorf("stop microsandbox VM before delete: %w", err) + } + } + if err := d.client.Remove(ctx, name); err != nil { + return fmt.Errorf("remove microsandbox VM: %w", err) + } + return nil +} + +func (d *microsandboxDriver) GetDevContainerLogs( + ctx context.Context, + workspaceID string, + stdout io.Writer, + stderr io.Writer, +) error { + return d.client.Logs(ctx, sandboxName(workspaceID), stdout) +} + +func (d *microsandboxDriver) runFromOptions( + ctx context.Context, + workspaceID string, + options *driver.RunOptions, +) error { + if options == nil { + return fmt.Errorf( + "microsandbox driver requires run options (in-place reprovision is unsupported)", + ) + } + if options.Image == "" { + return fmt.Errorf("microsandbox driver requires an image to run") + } + warnUnsupportedOptions(options) + if err := d.DeleteDevContainer(ctx, workspaceID); err != nil { + return fmt.Errorf("clean stale microsandbox VM before create: %w", err) + } + if err := d.client.EnsureImage(ctx, options.Image); err != nil { + log.Debugf("microsandbox ensure image %q failed (continuing): %v", options.Image, err) + } + if err := d.client.Create( + ctx, + sandboxName(workspaceID), + d.buildSpec(workspaceID, options), + ); err != nil { + return fmt.Errorf("create microsandbox VM: %w", err) + } + return nil +} + +func (d *microsandboxDriver) dockerImageDriver() (driver.ImageDriver, error) { + if d.docker != nil { + return d.docker, nil + } + if d.workspaceInfo == nil { + return nil, fmt.Errorf("microsandbox build requires workspace info") + } + dd, err := dockerdriver.NewDockerDriver(d.workspaceInfo) + if err != nil { + return nil, fmt.Errorf("microsandbox needs docker to build this devcontainer: %w", err) + } + d.docker = dd + return dd, nil +} + +func (d *microsandboxDriver) buildSpec(workspaceID string, options *driver.RunOptions) sandboxSpec { + labels := config.ListToObject(config.GetIDLabels(workspaceID, d.idLabels)) + if labels == nil { + labels = map[string]string{} + } + if options.User != "" { + labels[userLabel] = options.User + } + return sandboxSpec{ + Image: options.Image, + Entrypoint: entrypointArgv(options), + Memory: d.defaults.memory, + CPUs: d.defaults.cpus, + Env: options.Env, + Labels: labels, + Ephemeral: d.defaults.ephemeral, + IdleTimeout: d.defaults.idleTimeout, + Mounts: volumeMounts(options.Mounts), + MaxMemory: d.defaults.maxMemory, + MaxCPUs: d.defaults.maxCPUs, + BlockEgress: d.defaults.blockEgress, + } +} + +func volumeMounts(mounts []*config.Mount) []volumeMount { + var out []volumeMount + for _, m := range mounts { + if m == nil || m.Target == "" { + continue + } + switch m.Type { + case driver.MountTypeVolume: + out = append(out, volumeMount{Target: m.Target, Volume: m.Source}) + case driver.MountTypeTmpfs: + out = append(out, volumeMount{Target: m.Target, Tmpfs: true}) + } + } + return out +} + +func warnUnsupportedOptions(options *driver.RunOptions) { + var ignored []string + if isPrivileged(options) { + ignored = append(ignored, "privileged") + } + if len(options.CapAdd) > 0 { + ignored = append(ignored, "capAdd") + } + if len(options.SecurityOpt) > 0 { + ignored = append(ignored, "securityOpt") + } + if hasUserNSMapping(options) { + ignored = append(ignored, "user-namespace mapping") + } + if len(ignored) > 0 { + log.Warnf( + "microsandbox ignores container-runtime options not applicable to a microVM: %s", + strings.Join(ignored, ", "), + ) + } +} + +func isPrivileged(options *driver.RunOptions) bool { + return options.Privileged != nil && *options.Privileged +} + +func hasUserNSMapping(options *driver.RunOptions) bool { + return options.Userns != "" || len(options.UidMap) > 0 || len(options.GidMap) > 0 +} + +func entrypointArgv(options *driver.RunOptions) []string { + var argv []string + if options.Entrypoint != "" { + argv = append(argv, options.Entrypoint) + } + return append(argv, options.Cmd...) +} + +func toContainerDetails(info *sandboxInfo) *config.ContainerDetails { + status := "exited" + if info.Running { + status = "running" + } + return &config.ContainerDetails{ + ID: info.Name, + Created: info.CreatedAt.Format(createdAtLayout), + State: config.ContainerDetailsState{Status: status}, + Config: config.ContainerDetailsConfig{ + Labels: info.Labels, + User: info.Labels[userLabel], + }, + } +} + +const maxSandboxNameLen = 128 + +func sandboxName(workspaceID string) string { + name := "devsy-" + workspaceID + if len(name) <= maxSandboxNameLen { + return name + } + sum := sha256.Sum256([]byte(workspaceID)) + suffix := "-" + hex.EncodeToString(sum[:])[:12] + return name[:maxSandboxNameLen-len(suffix)] + suffix +} + +func parseUint32(s string) uint32 { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + v, err := strconv.ParseUint(s, 10, 32) + if err != nil { + log.Warnf("invalid microsandbox memory value %q, using runtime default", s) + return 0 + } + return uint32(v) +} + +func parseDuration(s string) time.Duration { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + d, err := time.ParseDuration(s) + if err != nil || d < 0 { + log.Warnf("invalid microsandbox inactivity timeout %q, disabling idle shutdown", s) + return 0 + } + return d +} + +func parseUint8(s string) uint8 { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + v, err := strconv.ParseUint(s, 10, 8) + if err != nil { + log.Warnf("invalid microsandbox cpus value %q, using runtime default", s) + return 0 + } + return uint8(v) +} diff --git a/pkg/driver/microsandbox/microsandbox_test.go b/pkg/driver/microsandbox/microsandbox_test.go new file mode 100644 index 000000000..59557235a --- /dev/null +++ b/pkg/driver/microsandbox/microsandbox_test.go @@ -0,0 +1,501 @@ +package microsandbox + +import ( + "bytes" + "context" + "errors" + "io" + "slices" + "strings" + "testing" + "time" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/driver" +) + +const ( + wsName = "devsy-ws1" + testImage = "example:latest" + testUser = "vscode" + imgX = "x:1" + callFind = "find:" + wsName + callRemove = "remove:" + wsName +) + +// fakeClient is an in-memory sandboxClient that records calls, so the driver's +// lifecycle logic can be tested without a live runtime. +type fakeClient struct { + created map[string]sandboxSpec + info map[string]*sandboxInfo + calls []string + execReq execRequest + execName string + failFind error + failStop error + failCreat error + failEnsure error +} + +func newFakeClient() *fakeClient { + return &fakeClient{created: map[string]sandboxSpec{}, info: map[string]*sandboxInfo{}} +} + +func (f *fakeClient) EnsureInstalled(context.Context) error { return nil } + +func (f *fakeClient) EnsureImage(_ context.Context, image string) error { + f.calls = append(f.calls, "ensure:"+image) + return f.failEnsure +} + +func (f *fakeClient) Create(_ context.Context, name string, spec sandboxSpec) error { + f.calls = append(f.calls, "create:"+name) + if f.failCreat != nil { + return f.failCreat + } + f.created[name] = spec + f.info[name] = &sandboxInfo{ + Name: name, + Running: true, + CreatedAt: time.Unix(0, 0), + Labels: spec.Labels, + } + return nil +} + +func (f *fakeClient) Find(_ context.Context, name string) (*sandboxInfo, error) { + f.calls = append(f.calls, "find:"+name) + if f.failFind != nil { + return nil, f.failFind + } + return f.info[name], nil +} + +func (f *fakeClient) Start(_ context.Context, name string) error { + f.calls = append(f.calls, "start:"+name) + return nil +} + +func (f *fakeClient) Stop(_ context.Context, name string) error { + f.calls = append(f.calls, "stop:"+name) + if f.failStop != nil { + return f.failStop + } + if info := f.info[name]; info != nil { + info.Running = false + } + return nil +} + +func (f *fakeClient) Remove(_ context.Context, name string) error { + f.calls = append(f.calls, "remove:"+name) + delete(f.info, name) + return nil +} + +func (f *fakeClient) Exec(_ context.Context, name string, req execRequest) error { + f.calls = append(f.calls, "exec:"+name) + f.execName = name + f.execReq = req + return nil +} + +func (f *fakeClient) Logs(_ context.Context, name string, _ io.Writer) error { + f.calls = append(f.calls, "logs:"+name) + return nil +} + +var _ sandboxClient = (*fakeClient)(nil) + +func TestRunDevContainerBuildsSpec(t *testing.T) { + f := newFakeClient() + d := newDriver(f, nil, specDefaults{memory: 2048, cpus: 4, ephemeral: true}) + + err := d.RunDevContainer(context.Background(), "ws1", &driver.RunOptions{ + Image: testImage, + User: testUser, + Env: map[string]string{"FOO": "bar"}, + }) + if err != nil { + t.Fatalf("RunDevContainer: %v", err) + } + + spec, ok := f.created[wsName] + if !ok { + t.Fatal("expected a sandbox named devsy-ws1 to be created") + } + got := struct { + image string + memory uint32 + cpus uint8 + ephemeral bool + }{spec.Image, spec.Memory, spec.CPUs, spec.Ephemeral} + want := struct { + image string + memory uint32 + cpus uint8 + ephemeral bool + }{testImage, 2048, 4, true} + if got != want { + t.Errorf("spec = %+v, want %+v", got, want) + } + if spec.Labels[userLabel] != testUser { + t.Errorf("user label = %q, want vscode", spec.Labels[userLabel]) + } + if spec.Env["FOO"] != "bar" { + t.Errorf("env not propagated: %+v", spec.Env) + } +} + +func TestRunDevContainerReplacesStaleSandbox(t *testing.T) { + f := newFakeClient() + f.info[wsName] = &sandboxInfo{Name: wsName, Running: true} + d := newDriver(f, nil, specDefaults{}) + + err := d.RunDevContainer(context.Background(), "ws1", &driver.RunOptions{Image: imgX}) + if err != nil { + t.Fatalf("RunDevContainer: %v", err) + } + want := []string{ + callFind, + "stop:devsy-ws1", + callRemove, + "ensure:x:1", + "create:devsy-ws1", + } + if !slices.Equal(f.calls, want) { + t.Errorf("call order = %v, want %v", f.calls, want) + } +} + +func TestRunDevContainerContinuesWhenPrePullFails(t *testing.T) { + f := newFakeClient() + f.failEnsure = errors.New("registry hiccup") + d := newDriver(f, nil, specDefaults{}) + + // Pre-pull failure must not abort the run; create should still be attempted. + if err := d.RunDevContainer( + context.Background(), + "ws1", + &driver.RunOptions{Image: imgX}, + ); err != nil { + t.Fatalf("RunDevContainer should proceed despite pull failure: %v", err) + } + if _, ok := f.created[wsName]; !ok { + t.Error("expected create to run even though pre-pull failed") + } +} + +func TestRunImageDevContainerRunsFromParams(t *testing.T) { + f := newFakeClient() + d := newDriver(f, nil, specDefaults{}) + + err := d.RunImageDevContainer(context.Background(), &driver.RunImageDevContainerParams{ + WorkspaceID: "ws1", + Options: &driver.RunOptions{Image: "built-local:latest"}, + }) + if err != nil { + t.Fatalf("RunImageDevContainer: %v", err) + } + want := []string{callFind, "ensure:built-local:latest", "create:devsy-ws1"} + if !slices.Equal(f.calls, want) { + t.Errorf("call order = %v, want %v", f.calls, want) + } +} + +func TestCheckGPURequirement(t *testing.T) { + // required GPU -> error + req := &config.DevContainerConfig{} + req.HostRequirements = &config.HostRequirements{GPU: &config.GPURequirement{Value: "true"}} + if err := checkGPURequirement(req); err == nil { + t.Error("expected an error when a GPU is required") + } + // optional GPU -> no error + opt := &config.DevContainerConfig{} + opt.HostRequirements = &config.HostRequirements{GPU: &config.GPURequirement{Value: "optional"}} + if err := checkGPURequirement(opt); err != nil { + t.Errorf("optional GPU should not error, got %v", err) + } + // no GPU / nil config -> no error + if err := checkGPURequirement(nil); err != nil { + t.Errorf("nil config should not error, got %v", err) + } + if err := checkGPURequirement(&config.DevContainerConfig{}); err != nil { + t.Errorf("no host requirements should not error, got %v", err) + } +} + +func TestUpdateContainerUserUIDIsNoop(t *testing.T) { + d := newDriver(newFakeClient(), nil, specDefaults{}) + if err := d.UpdateContainerUserUID(context.Background(), "ws1", nil, nil); err != nil { + t.Errorf("UpdateContainerUserUID should be a no-op, got %v", err) + } +} + +func TestRunDevContainerRequiresImage(t *testing.T) { + d := newDriver(newFakeClient(), nil, specDefaults{}) + if err := d.RunDevContainer(context.Background(), "ws1", &driver.RunOptions{}); err == nil { + t.Fatal("expected an error when image is empty") + } +} + +func TestFindDevContainerMapsState(t *testing.T) { + f := newFakeClient() + f.info[wsName] = &sandboxInfo{ + Name: wsName, + Running: true, + CreatedAt: time.Unix(0, 0), + Labels: map[string]string{userLabel: testUser}, + } + d := newDriver(f, nil, specDefaults{}) + + details, err := d.FindDevContainer(context.Background(), "ws1") + if err != nil { + t.Fatalf("FindDevContainer: %v", err) + } + if details == nil { + t.Fatal("expected details for an existing sandbox") + } + if details.State.Status != "running" { + t.Errorf("status = %q, want running", details.State.Status) + } + if details.Config.User != testUser { + t.Errorf("user = %q, want vscode", details.Config.User) + } +} + +func TestFindDevContainerAbsent(t *testing.T) { + d := newDriver(newFakeClient(), nil, specDefaults{}) + details, err := d.FindDevContainer(context.Background(), "missing") + if err != nil { + t.Fatalf("FindDevContainer: %v", err) + } + if details != nil { + t.Errorf("expected nil details for a missing sandbox, got %+v", details) + } +} + +func TestDeleteStopsRunningThenRemoves(t *testing.T) { + f := newFakeClient() + f.info[wsName] = &sandboxInfo{Name: wsName, Running: true} + d := newDriver(f, nil, specDefaults{}) + + if err := d.DeleteDevContainer(context.Background(), "ws1"); err != nil { + t.Fatalf("DeleteDevContainer: %v", err) + } + want := []string{callFind, "stop:devsy-ws1", callRemove} + if !slices.Equal(f.calls, want) { + t.Errorf("call order = %v, want %v", f.calls, want) + } +} + +func TestDeleteStoppedSkipsStop(t *testing.T) { + f := newFakeClient() + f.info[wsName] = &sandboxInfo{Name: wsName, Running: false} + d := newDriver(f, nil, specDefaults{}) + + if err := d.DeleteDevContainer(context.Background(), "ws1"); err != nil { + t.Fatalf("DeleteDevContainer: %v", err) + } + want := []string{callFind, callRemove} + if !slices.Equal(f.calls, want) { + t.Errorf("call order = %v, want %v", f.calls, want) + } +} + +func TestDeleteAbsentIsNoop(t *testing.T) { + f := newFakeClient() + d := newDriver(f, nil, specDefaults{}) + if err := d.DeleteDevContainer(context.Background(), "missing"); err != nil { + t.Fatalf("DeleteDevContainer: %v", err) + } + want := []string{"find:devsy-missing"} + if !slices.Equal(f.calls, want) { + t.Errorf("calls = %v, want just a find", f.calls) + } +} + +func TestDeletePropagatesStopError(t *testing.T) { + f := newFakeClient() + f.info[wsName] = &sandboxInfo{Name: wsName, Running: true} + f.failStop = errors.New("boom") + d := newDriver(f, nil, specDefaults{}) + + if err := d.DeleteDevContainer(context.Background(), "ws1"); err == nil { + t.Fatal("expected the stop error to propagate") + } +} + +func TestCommandDevContainerForwardsRequest(t *testing.T) { + f := newFakeClient() + d := newDriver(f, nil, specDefaults{}) + var out bytes.Buffer + + err := d.CommandDevContainer(context.Background(), &driver.CommandParams{ + WorkspaceID: "ws1", + User: testUser, + Command: "echo hi", + Stdout: &out, + }) + if err != nil { + t.Fatalf("CommandDevContainer: %v", err) + } + if f.execName != wsName { + t.Errorf("exec name = %q, want devsy-ws1", f.execName) + } + if f.execReq.Command != "echo hi" || f.execReq.User != testUser { + t.Errorf("unexpected exec request: %+v", f.execReq) + } +} + +func TestRunDevContainerSetsEntrypoint(t *testing.T) { + f := newFakeClient() + d := newDriver(f, nil, specDefaults{}) + + err := d.RunDevContainer(context.Background(), "ws1", &driver.RunOptions{ + Image: testImage, + Entrypoint: "/bin/sh", + Cmd: []string{"-c", "start", "-"}, + }) + if err != nil { + t.Fatalf("RunDevContainer: %v", err) + } + got := f.created[wsName].Entrypoint + want := []string{"/bin/sh", "-c", "start", "-"} + if len(got) != len(want) { + t.Fatalf("entrypoint = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("entrypoint = %v, want %v", got, want) + } + } +} + +func TestCommandContainerArgvForwardsArgv(t *testing.T) { + f := newFakeClient() + d := newDriver(f, nil, specDefaults{}) + + argv := []string{"sh", "-c", "cat > /usr/local/bin/devsy"} + err := d.CommandContainerArgv(context.Background(), "ws1", argv, driver.Streams{}) + if err != nil { + t.Fatalf("CommandContainerArgv: %v", err) + } + if f.execName != wsName { + t.Errorf("exec name = %q, want devsy-ws1", f.execName) + } + if len(f.execReq.Argv) != len(argv) || f.execReq.Argv[0] != "sh" { + t.Errorf("argv = %v, want %v", f.execReq.Argv, argv) + } + if f.execReq.Command != "" { + t.Errorf("argv exec must not set a shell command, got %q", f.execReq.Command) + } + if f.execReq.User != "root" { + t.Errorf("argv exec user = %q, want root", f.execReq.User) + } +} + +func TestSandboxName(t *testing.T) { + if got := sandboxName("my-workspace"); got != "devsy-my-workspace" { + t.Errorf("sandboxName = %q, want %q", got, "devsy-my-workspace") + } +} + +func TestSandboxNameClampsLongIDs(t *testing.T) { + long := strings.Repeat("a", 200) + got := sandboxName(long) + if len(got) != maxSandboxNameLen { + t.Errorf("clamped name length = %d, want %d", len(got), maxSandboxNameLen) + } + // Distinct long ids must not collide after clamping. + other := sandboxName(strings.Repeat("a", 199) + "b") + if got == other { + t.Error("distinct long ids produced the same clamped name") + } + // Short ids stay verbatim. + if sandboxName("x") != "devsy-x" { + t.Errorf("short id should not be clamped, got %q", sandboxName("x")) + } +} + +func TestParseUint32(t *testing.T) { + cases := []struct { + in string + want uint32 + }{{"", 0}, {" ", 0}, {"2048", 2048}, {"not-num", 0}, {"-5", 0}} + for _, c := range cases { + if got := parseUint32(c.in); got != c.want { + t.Errorf("parseUint32(%q) = %d, want %d", c.in, got, c.want) + } + } +} + +func TestParseDuration(t *testing.T) { + cases := map[string]time.Duration{ + "": 0, + " ": 0, + "20s": 20 * time.Second, + "1h": time.Hour, + "bogus": 0, + "-5m": 0, + } + for in, want := range cases { + if got := parseDuration(in); got != want { + t.Errorf("parseDuration(%q) = %s, want %s", in, got, want) + } + } +} + +func TestBuildSpecMapsVolumeMountsSkipsBind(t *testing.T) { + d := newDriver(newFakeClient(), nil, specDefaults{}) + spec := d.buildSpec("ws1", &driver.RunOptions{ + Image: imgX, + Mounts: []*config.Mount{ + {Type: driver.MountTypeVolume, Source: "vol1", Target: "/data"}, + {Type: driver.MountTypeTmpfs, Target: "/scratch"}, + {Type: driver.MountTypeBind, Source: "/host", Target: "/mnt"}, + nil, + }, + }) + if len(spec.Mounts) != 2 { + t.Fatalf("got %d mounts, want 2 (bind + nil skipped): %+v", len(spec.Mounts), spec.Mounts) + } + if spec.Mounts[0].Target != "/data" || spec.Mounts[0].Volume != "vol1" || spec.Mounts[0].Tmpfs { + t.Errorf("volume mount mapped wrong: %+v", spec.Mounts[0]) + } + if spec.Mounts[1].Target != "/scratch" || !spec.Mounts[1].Tmpfs { + t.Errorf("tmpfs mount mapped wrong: %+v", spec.Mounts[1]) + } +} + +func TestBuildSpecCarriesIdleTimeout(t *testing.T) { + d := newDriver(newFakeClient(), nil, specDefaults{idleTimeout: 90 * time.Second}) + spec := d.buildSpec("ws1", &driver.RunOptions{Image: imgX}) + if spec.IdleTimeout != 90*time.Second { + t.Errorf("idle timeout = %s, want 90s", spec.IdleTimeout) + } +} + +func TestBuildSpecCarriesCeilingsAndEgress(t *testing.T) { + d := newDriver( + newFakeClient(), + nil, + specDefaults{maxMemory: 4096, maxCPUs: 4, blockEgress: true}, + ) + spec := d.buildSpec("ws1", &driver.RunOptions{Image: imgX}) + if spec.MaxMemory != 4096 || spec.MaxCPUs != 4 || !spec.BlockEgress { + t.Errorf("unexpected spec ceilings/egress: %+v", spec) + } +} + +func TestParseUint8(t *testing.T) { + cases := []struct { + in string + want uint8 + }{{"", 0}, {"2", 2}, {"999", 0}, {"abc", 0}, {" 4 ", 4}} + for _, c := range cases { + if got := parseUint8(c.in); got != c.want { + t.Errorf("parseUint8(%q) = %d, want %d", c.in, got, c.want) + } + } +} diff --git a/pkg/options/resolve.go b/pkg/options/resolve.go index 80729caa3..aabef7d68 100644 --- a/pkg/options/resolve.go +++ b/pkg/options/resolve.go @@ -254,6 +254,7 @@ func ResolveAgentConfig( resolveAgentDockerConfig(&agentConfig, options) resolveAgentKubernetesConfig(&agentConfig, options) resolveAgentAppleConfig(&agentConfig, options) + resolveAgentMicrosandboxConfig(&agentConfig, options) resolveAgentPathAndURL(&agentConfig, options, devConfig) resolveAgentCredentials(&agentConfig, options, devConfig) @@ -342,6 +343,34 @@ func resolveAgentAppleConfig( agentConfig.Apple.Env = resolver.ResolveDefaultValues(agentConfig.Apple.Env, options) } +func resolveAgentMicrosandboxConfig( + agentConfig *provider.ProviderAgentConfig, + options map[string]string, +) { + agentConfig.Microsandbox.Memory = resolver.ResolveDefaultValue( + agentConfig.Microsandbox.Memory, + options, + ) + agentConfig.Microsandbox.CPUs = resolver.ResolveDefaultValue( + agentConfig.Microsandbox.CPUs, + options, + ) + agentConfig.Microsandbox.MaxMemory = resolver.ResolveDefaultValue( + agentConfig.Microsandbox.MaxMemory, + options, + ) + agentConfig.Microsandbox.MaxCPUs = resolver.ResolveDefaultValue( + agentConfig.Microsandbox.MaxCPUs, + options, + ) + agentConfig.Microsandbox.Ephemeral = types.StrBool( + resolver.ResolveDefaultValue(string(agentConfig.Microsandbox.Ephemeral), options), + ) + agentConfig.Microsandbox.BlockEgress = types.StrBool( + resolver.ResolveDefaultValue(string(agentConfig.Microsandbox.BlockEgress), options), + ) +} + func resolveAgentPathAndURL( agentConfig *provider.ProviderAgentConfig, options map[string]string, diff --git a/pkg/options/resolve_test.go b/pkg/options/resolve_test.go index 9a4347e9b..aeecfa647 100644 --- a/pkg/options/resolve_test.go +++ b/pkg/options/resolve_test.go @@ -751,3 +751,29 @@ func TestResolveAgentAppleConfig(t *testing.T) { assert.Equal(t, types.StrBool("true"), agentConfig.Apple.Rosetta) assert.Equal(t, "resolved", agentConfig.Apple.Env["KEY"]) } + +func TestResolveAgentMicrosandboxConfig(t *testing.T) { + agentConfig := &provider.ProviderAgentConfig{} + agentConfig.Microsandbox.Memory = "${MICROSANDBOX_MEMORY}" + agentConfig.Microsandbox.CPUs = "${MICROSANDBOX_CPUS}" + agentConfig.Microsandbox.Ephemeral = types.StrBool("${MICROSANDBOX_EPHEMERAL}") + + agentConfig.Microsandbox.MaxMemory = "${MICROSANDBOX_MAX_MEMORY}" + agentConfig.Microsandbox.BlockEgress = types.StrBool("${MICROSANDBOX_BLOCK_EGRESS}") + + options := map[string]string{ + "MICROSANDBOX_MEMORY": "2048", + "MICROSANDBOX_CPUS": "4", + "MICROSANDBOX_EPHEMERAL": "true", + "MICROSANDBOX_MAX_MEMORY": "8192", + "MICROSANDBOX_BLOCK_EGRESS": "true", + } + + resolveAgentMicrosandboxConfig(agentConfig, options) + + assert.Equal(t, "2048", agentConfig.Microsandbox.Memory) + assert.Equal(t, "4", agentConfig.Microsandbox.CPUs) + assert.Equal(t, types.StrBool("true"), agentConfig.Microsandbox.Ephemeral) + assert.Equal(t, "8192", agentConfig.Microsandbox.MaxMemory) + assert.Equal(t, types.StrBool("true"), agentConfig.Microsandbox.BlockEgress) +} diff --git a/pkg/provider/parse.go b/pkg/provider/parse.go index f01f9c39e..4c47718cb 100644 --- a/pkg/provider/parse.go +++ b/pkg/provider/parse.go @@ -280,8 +280,11 @@ func validateAgentDriver(config *ProviderConfig) error { if config.Agent.Driver != "" && config.Agent.Driver != CustomDriver && config.Agent.Driver != DockerDriver && config.Agent.Driver != KubernetesDriver && - config.Agent.Driver != AppleDriver { - return fmt.Errorf("agent.driver can only be docker, kubernetes, apple or custom") + config.Agent.Driver != AppleDriver && + config.Agent.Driver != MicrosandboxDriver { + return fmt.Errorf( + "agent.driver can only be docker, kubernetes, apple, microsandbox or custom", + ) } if config.Agent.Driver == CustomDriver { diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 1cdadb0c1..d86e92fb5 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -127,6 +127,9 @@ type ProviderAgentConfig struct { // Apple holds Apple container specific configuration Apple ProviderAppleDriverConfig `json:"apple"` + + // Microsandbox holds microsandbox microVM specific configuration + Microsandbox ProviderMicrosandboxDriverConfig `json:"microsandbox"` } type ProviderDockerlessOptions struct { @@ -151,10 +154,11 @@ func (a ProviderAgentConfig) IsDockerDriver() bool { } const ( - DockerDriver = "docker" - KubernetesDriver = "kubernetes" - CustomDriver = "custom" - AppleDriver = "apple" + DockerDriver = "docker" + KubernetesDriver = "kubernetes" + CustomDriver = "custom" + AppleDriver = "apple" + MicrosandboxDriver = "microsandbox" ) // ProviderAppleDriverConfig holds configuration for the Apple container driver, @@ -170,6 +174,29 @@ type ProviderAppleDriverConfig struct { Env map[string]string `json:"env,omitempty"` } +// ProviderMicrosandboxDriverConfig holds configuration for the microsandbox +// driver, which boots the devcontainer OCI image as a hardware-isolated microVM +// (libkrun) via the microsandbox runtime. +type ProviderMicrosandboxDriverConfig struct { + // Memory is the guest memory limit in MiB. Empty uses the runtime default. + Memory string `json:"memory,omitempty"` + + // CPUs is the number of virtual CPUs. Empty uses the runtime default. + CPUs string `json:"cpus,omitempty"` + + // MaxMemory is the hotplug memory ceiling in MiB. Empty uses the default. + MaxMemory string `json:"maxMemory,omitempty"` + + // MaxCPUs is the hotplug CPU ceiling. Empty uses the default. + MaxCPUs string `json:"maxCpus,omitempty"` + + // BlockEgress denies outbound public network (sandbox hardening). + BlockEgress types.StrBool `json:"blockEgress,omitempty"` + + // Ephemeral removes the sandbox's disk state when it stops. + Ephemeral types.StrBool `json:"ephemeral,omitempty"` +} + type ProviderCustomDriverConfig struct { // FindDevContainer is used to find an existing devcontainer FindDevContainer types.StrArray `json:"findDevContainer,omitempty"` diff --git a/providers/microsandbox/provider.yaml b/providers/microsandbox/provider.yaml new file mode 100644 index 000000000..6c432d3de --- /dev/null +++ b/providers/microsandbox/provider.yaml @@ -0,0 +1,49 @@ +name: microsandbox +version: v1.0.0 +icon: https://dl.devsy.sh/assets/devsy.svg +home: https://github.com/devsy-org/devsy +description: |- + Devsy on microsandbox (hardware-isolated microVMs, libkrun) +optionGroups: + - options: + - MICROSANDBOX_MEMORY + - MICROSANDBOX_CPUS + - MICROSANDBOX_MAX_MEMORY + - MICROSANDBOX_MAX_CPUS + - MICROSANDBOX_BLOCK_EGRESS + - MICROSANDBOX_EPHEMERAL + - INACTIVITY_TIMEOUT + name: "Advanced Options" +options: + MICROSANDBOX_MEMORY: + description: "Guest memory limit in MiB. Empty uses the runtime default." + MICROSANDBOX_CPUS: + description: "Number of virtual CPUs. Empty uses the runtime default." + MICROSANDBOX_MAX_MEMORY: + description: "Hotplug memory ceiling in MiB. Empty uses the runtime default." + MICROSANDBOX_MAX_CPUS: + description: "Hotplug CPU ceiling. Empty uses the runtime default." + MICROSANDBOX_BLOCK_EGRESS: + description: "If true, deny the microVM outbound public network (sandbox hardening)." + default: "false" + type: boolean + MICROSANDBOX_EPHEMERAL: + description: "If true, the microVM's disk state is discarded when it stops." + default: "false" + type: boolean + INACTIVITY_TIMEOUT: + description: "If defined, will automatically stop the microVM after the inactivity period. Examples: 10m, 1h" +agent: + containerInactivityTimeout: ${INACTIVITY_TIMEOUT} + local: true + driver: microsandbox + microsandbox: + memory: ${MICROSANDBOX_MEMORY} + cpus: ${MICROSANDBOX_CPUS} + maxMemory: ${MICROSANDBOX_MAX_MEMORY} + maxCpus: ${MICROSANDBOX_MAX_CPUS} + blockEgress: ${MICROSANDBOX_BLOCK_EGRESS} + ephemeral: ${MICROSANDBOX_EPHEMERAL} +exec: + command: |- + "${DEVSY}" internal sh -c "${COMMAND}" diff --git a/providers/providers.go b/providers/providers.go index d1dbe5a27..ff8c00cf7 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -13,6 +13,9 @@ var DockerProvider string //go:embed kubernetes/provider.yaml var KubernetesProvider string +//go:embed microsandbox/provider.yaml +var MicrosandboxProvider string + //go:embed podman/provider.yaml var PodmanProvider string @@ -22,10 +25,11 @@ var ProProvider string // GetBuiltInProviders retrieves the built in providers. func GetBuiltInProviders() map[string]string { return map[string]string{ - "apple": AppleProvider, - "docker": DockerProvider, - "kubernetes": KubernetesProvider, - "podman": PodmanProvider, - "pro": ProProvider, + "apple": AppleProvider, + "docker": DockerProvider, + "kubernetes": KubernetesProvider, + "microsandbox": MicrosandboxProvider, + "podman": PodmanProvider, + "pro": ProProvider, } } diff --git a/providers/providers_test.go b/providers/providers_test.go index aef5993e7..fe0b8d280 100644 --- a/providers/providers_test.go +++ b/providers/providers_test.go @@ -33,3 +33,16 @@ func TestAppleProviderUsesAppleDriver(t *testing.T) { t.Errorf("apple provider driver = %q, want %q", cfg.Agent.Driver, provider.AppleDriver) } } + +func TestMicrosandboxProviderUsesMicrosandboxDriver(t *testing.T) { + cfg, err := provider.ParseProvider(strings.NewReader(providers.MicrosandboxProvider)) + if err != nil { + t.Fatalf("parse microsandbox provider: %v", err) + } + if cfg.Agent.Driver != provider.MicrosandboxDriver { + t.Errorf( + "microsandbox provider driver = %q, want %q", + cfg.Agent.Driver, provider.MicrosandboxDriver, + ) + } +} From ab229dfa1989aba8d77dd5f97c976e8ceb8ea7f4 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 26 Jul 2026 21:32:59 -0500 Subject: [PATCH 2/5] fix(credentials): return 200 on the readiness probe path The #765 handler refactor switched unmatched paths from an implicit 200 (the old switch had no default) to http.NotFound. The readiness probe in waitForServer hits the root path, so startup detection began failing with 404 and git/docker credential forwarding was silently skipped for every driver ("credentials server did not start in time"). Add an explicit root route that returns 200 while keeping 404 for unknown credential endpoints. --- pkg/credentials/server.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/credentials/server.go b/pkg/credentials/server.go index 03a8fd4a4..5e3c37377 100644 --- a/pkg/credentials/server.go +++ b/pkg/credentials/server.go @@ -79,6 +79,12 @@ type credentialsHandlerFunc func( func newCredentialsHandler(ctx context.Context, client CredentialsClient) http.Handler { routes := map[string]credentialsHandlerFunc{ + // Root is a readiness probe (see waitForServer); it must return 200 so the + // server is detected as up. Unknown paths still 404 below. + "/": func(_ context.Context, writer http.ResponseWriter, _ *http.Request, _ CredentialsClient) error { + writer.WriteHeader(http.StatusOK) + return nil + }, "/git-credentials": handleGitCredentialsRequest, "/docker-credentials": handleDockerCredentialsRequest, "/git-ssh-signature": handleGitSSHSignatureRequest, From 793fc7fdae7d8fa18118fa38d61b49e6ac24dcec Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 26 Jul 2026 21:53:37 -0500 Subject: [PATCH 3/5] fix(driver): harden microsandbox Find and volume error handling Address review feedback: - Find now surfaces real inspect failures (permission, crash, bad invocation) instead of masking every error as "sandbox absent"; only a verified "sandbox not found" from msb inspect maps to (nil, nil). - ensureVolumes matches the specific "already exists" wording rather than a broad "exist" substring, so genuine volume-create failures are not swallowed. - Add direct unit tests for the pure arg-building and env-redaction helpers. --- pkg/driver/microsandbox/cliclient.go | 26 ++-- pkg/driver/microsandbox/cliclient_test.go | 124 +++++++++++++++++++ pkg/driver/microsandbox/microsandbox_test.go | 6 +- 3 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 pkg/driver/microsandbox/cliclient_test.go diff --git a/pkg/driver/microsandbox/cliclient.go b/pkg/driver/microsandbox/cliclient.go index 894ee20d3..a7e97e40a 100644 --- a/pkg/driver/microsandbox/cliclient.go +++ b/pkg/driver/microsandbox/cliclient.go @@ -3,6 +3,7 @@ package microsandbox import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -22,6 +23,12 @@ type cliClient struct{} var _ sandboxClient = cliClient{} +const ( + cmdCreate = "create" + flagName = "--name" + flagEnv = "--env" +) + func (cliClient) EnsureInstalled(_ context.Context) error { bin := msbBinary() if filepath.IsAbs(bin) { @@ -63,12 +70,17 @@ func (cliClient) Find(ctx context.Context, sandbox string) (*sandboxInfo, error) out, err := exec.CommandContext(ctx, msbBinary(), "inspect", sandbox, "--format", "json"). Output() if err != nil { - // inspect fails when the sandbox is absent, which is not an error here. - // A cancelled/timed-out context is, so do not mask it as "not found". if ctx.Err() != nil { return nil, fmt.Errorf("inspect microsandbox VM %q: %w", sandbox, ctx.Err()) } - return nil, nil + // A genuine "not found" means the sandbox is absent (nil, nil). Any other + // failure (permission, crash, bad invocation) is a real error to surface, + // so callers do not mistake it for an absent sandbox. + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && strings.Contains(string(exitErr.Stderr), "not found") { + return nil, nil + } + return nil, fmt.Errorf("inspect microsandbox VM %q: %w", sandbox, err) } var raw struct { Name string `json:"name"` @@ -135,7 +147,7 @@ func (cliClient) ensureVolumes(ctx context.Context, mounts []volumeMount) error for _, vol := range namedVolumes(mounts) { // #nosec G204 -- args are a resolved binary path and a named-volume name out, err := exec.CommandContext(ctx, msbBinary(), "volume", "create", vol).CombinedOutput() - if err != nil && !strings.Contains(strings.ToLower(string(out)), "exist") { + if err != nil && !strings.Contains(strings.ToLower(string(out)), "already exists") { return fmt.Errorf("create microsandbox volume %q: %s: %w", vol, out, err) } } @@ -143,7 +155,7 @@ func (cliClient) ensureVolumes(ctx context.Context, mounts []volumeMount) error } func createArgs(sandbox string, spec sandboxSpec) []string { - args := []string{"create", "--name", sandbox} + args := []string{cmdCreate, flagName, sandbox} args = append(args, runtimeArgs(spec)...) args = append(args, resourceArgs(spec)...) args = append(args, mountArgs(spec.Mounts)...) @@ -156,7 +168,7 @@ func runtimeArgs(spec sandboxSpec) []string { args = append(args, "--entrypoint", strings.Join(spec.Entrypoint, " ")) } for k, v := range spec.Env { - args = append(args, "--env", k+"="+v) + args = append(args, flagEnv, k+"="+v) } for k, v := range spec.Labels { args = append(args, "--label", k+"="+v) @@ -230,7 +242,7 @@ func redactArgs(args []string) string { out := make([]string, len(args)) copy(out, args) for i := 0; i+1 < len(out); i++ { - if out[i] != "--env" { + if out[i] != flagEnv { continue } if k, _, ok := strings.Cut(out[i+1], "="); ok { diff --git a/pkg/driver/microsandbox/cliclient_test.go b/pkg/driver/microsandbox/cliclient_test.go new file mode 100644 index 000000000..bfee25e57 --- /dev/null +++ b/pkg/driver/microsandbox/cliclient_test.go @@ -0,0 +1,124 @@ +package microsandbox + +import ( + "slices" + "strings" + "testing" + "time" +) + +func TestCreateArgsFull(t *testing.T) { + spec := sandboxSpec{ + Image: testImg, + Entrypoint: []string{shPath, "-c", "sleep infinity"}, + Env: map[string]string{"K": "v"}, + Labels: map[string]string{"devsy.sh/user": "vscode"}, + IdleTimeout: 90 * time.Second, + BlockEgress: true, + Memory: 2048, + MaxMemory: 4096, + CPUs: 2, + MaxCPUs: 4, + Mounts: []volumeMount{ + {Target: "/cache", Volume: "cache-vol"}, + {Target: "/tmp", Tmpfs: true}, + }, + } + args := createArgs(wsName, spec) + + // name comes first, image last. + if args[0] != cmdCreate || args[1] != flagName || args[2] != wsName { + t.Errorf("prefix = %v", args[:3]) + } + if args[len(args)-1] != testImg { + t.Errorf("image should be the final arg, got %q", args[len(args)-1]) + } + + want := [][2]string{ + {"--entrypoint", "/bin/sh -c sleep infinity"}, + {flagEnv, "K=v"}, + {"--label", "devsy.sh/user=vscode"}, + {"--idle-timeout", "1m30s"}, + {"--memory", "2048M"}, + {"--max-memory", "4096M"}, + {"--cpus", "2"}, + {"--max-cpus", "4"}, + {"--mount-named", "cache-vol:/cache"}, + {"--tmpfs", "/tmp"}, + } + for _, kv := range want { + if !hasFlagValue(args, kv[0], kv[1]) { + t.Errorf("missing %s %q in %v", kv[0], kv[1], args) + } + } + if hasFlag(args, "--net-default-egress") == false { + t.Errorf("expected egress deny flag in %v", args) + } +} + +func TestCreateArgsMinimal(t *testing.T) { + args := createArgs(wsName, sandboxSpec{Image: testImg}) + // Only name + image; no sizing/runtime flags for a bare spec. + want := []string{cmdCreate, flagName, wsName, testImg} + if !slices.Equal(args, want) { + t.Errorf("args = %v, want %v", args, want) + } +} + +func TestMountArgsAndNamedVolumes(t *testing.T) { + mounts := []volumeMount{ + {Target: "/a", Volume: "vol-a"}, + {Target: "/b", Tmpfs: true}, + {Target: "/c", Volume: "vol-c"}, + } + if got := namedVolumes(mounts); !slices.Equal(got, []string{"vol-a", "vol-c"}) { + t.Errorf("namedVolumes = %v, want [vol-a vol-c]", got) + } + args := mountArgs(mounts) + if !hasFlagValue(args, "--mount-named", "vol-a:/a") || + !hasFlagValue(args, "--tmpfs", "/b") || + !hasFlagValue(args, "--mount-named", "vol-c:/c") { + t.Errorf("mountArgs = %v", args) + } +} + +func TestResourceArgsOmitsZero(t *testing.T) { + if got := resourceArgs(sandboxSpec{}); len(got) != 0 { + t.Errorf("zero spec should produce no resource args, got %v", got) + } + if got := resourceArgs( + sandboxSpec{Memory: 512}, + ); !slices.Equal( + got, + []string{"--memory", "512M"}, + ) { + t.Errorf("resourceArgs = %v", got) + } +} + +func TestRedactArgsMasksEnvValues(t *testing.T) { + args := []string{cmdCreate, flagEnv, "TOKEN=s3cret", "--label", "k=v", flagEnv, "PLAIN=ok"} + got := redactArgs(args) + if strings.Contains(got, "s3cret") || strings.Contains(got, "ok") { + t.Errorf("env values leaked: %q", got) + } + if !strings.Contains(got, "TOKEN=***") || !strings.Contains(got, "PLAIN=***") { + t.Errorf("env keys should be preserved with masked values: %q", got) + } + if !strings.Contains(got, "--label k=v") { + t.Errorf("non-env args should be untouched: %q", got) + } +} + +func hasFlag(args []string, flag string) bool { + return slices.Contains(args, flag) +} + +func hasFlagValue(args []string, flag, value string) bool { + for i := 0; i+1 < len(args); i++ { + if args[i] == flag && args[i+1] == value { + return true + } + } + return false +} diff --git a/pkg/driver/microsandbox/microsandbox_test.go b/pkg/driver/microsandbox/microsandbox_test.go index 59557235a..d4fc75d3b 100644 --- a/pkg/driver/microsandbox/microsandbox_test.go +++ b/pkg/driver/microsandbox/microsandbox_test.go @@ -19,6 +19,8 @@ const ( testImage = "example:latest" testUser = "vscode" imgX = "x:1" + testImg = "img:1" + shPath = "/bin/sh" callFind = "find:" + wsName callRemove = "remove:" + wsName ) @@ -354,14 +356,14 @@ func TestRunDevContainerSetsEntrypoint(t *testing.T) { err := d.RunDevContainer(context.Background(), "ws1", &driver.RunOptions{ Image: testImage, - Entrypoint: "/bin/sh", + Entrypoint: shPath, Cmd: []string{"-c", "start", "-"}, }) if err != nil { t.Fatalf("RunDevContainer: %v", err) } got := f.created[wsName].Entrypoint - want := []string{"/bin/sh", "-c", "start", "-"} + want := []string{shPath, "-c", "start", "-"} if len(got) != len(want) { t.Fatalf("entrypoint = %v, want %v", got, want) } From 92aa5639a05810ec4c5e9b80cd97aab6a3479fab Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 26 Jul 2026 21:58:54 -0500 Subject: [PATCH 4/5] refactor(driver): use pkg/flags/names for microsandbox flag refs Reference the shared names package (names.Create, names.Flag(names.Name/ Env/Label/User)) instead of ad-hoc local flag constants; the microsandbox-specific flags without a shared name stay as literals. --- pkg/driver/microsandbox/cliclient.go | 24 ++++++++++------------- pkg/driver/microsandbox/cliclient_test.go | 18 +++++++++++++---- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/pkg/driver/microsandbox/cliclient.go b/pkg/driver/microsandbox/cliclient.go index a7e97e40a..0585cbdde 100644 --- a/pkg/driver/microsandbox/cliclient.go +++ b/pkg/driver/microsandbox/cliclient.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/devsy-org/devsy/pkg/flags/names" "github.com/devsy-org/devsy/pkg/image" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/tarball" @@ -23,12 +24,6 @@ type cliClient struct{} var _ sandboxClient = cliClient{} -const ( - cmdCreate = "create" - flagName = "--name" - flagEnv = "--env" -) - func (cliClient) EnsureInstalled(_ context.Context) error { bin := msbBinary() if filepath.IsAbs(bin) { @@ -119,7 +114,7 @@ func (cliClient) Remove(ctx context.Context, sandbox string) error { func (cliClient) Exec(ctx context.Context, sandbox string, req execRequest) error { args := []string{"exec", "--stream"} if req.User != "" { - args = append(args, "--user", req.User) + args = append(args, names.Flag(names.User), req.User) } args = append(args, sandbox, "--") if len(req.Argv) > 0 { @@ -155,7 +150,7 @@ func (cliClient) ensureVolumes(ctx context.Context, mounts []volumeMount) error } func createArgs(sandbox string, spec sandboxSpec) []string { - args := []string{cmdCreate, flagName, sandbox} + args := []string{names.Create, names.Flag(names.Name), sandbox} args = append(args, runtimeArgs(spec)...) args = append(args, resourceArgs(spec)...) args = append(args, mountArgs(spec.Mounts)...) @@ -168,10 +163,10 @@ func runtimeArgs(spec sandboxSpec) []string { args = append(args, "--entrypoint", strings.Join(spec.Entrypoint, " ")) } for k, v := range spec.Env { - args = append(args, flagEnv, k+"="+v) + args = append(args, names.Flag(names.Env), k+"="+v) } for k, v := range spec.Labels { - args = append(args, "--label", k+"="+v) + args = append(args, names.Flag(names.Label), k+"="+v) } if spec.IdleTimeout > 0 { args = append(args, "--idle-timeout", spec.IdleTimeout.String()) @@ -213,13 +208,13 @@ func mountArgs(mounts []volumeMount) []string { } func namedVolumes(mounts []volumeMount) []string { - var names []string + var vols []string for _, m := range mounts { if !m.Tmpfs && m.Volume != "" { - names = append(names, m.Volume) + vols = append(vols, m.Volume) } } - return names + return vols } func msbRun(ctx context.Context, args ...string) error { @@ -241,8 +236,9 @@ func msbRun(ctx context.Context, args ...string) error { func redactArgs(args []string) string { out := make([]string, len(args)) copy(out, args) + envFlag := names.Flag(names.Env) for i := 0; i+1 < len(out); i++ { - if out[i] != flagEnv { + if out[i] != envFlag { continue } if k, _, ok := strings.Cut(out[i+1], "="); ok { diff --git a/pkg/driver/microsandbox/cliclient_test.go b/pkg/driver/microsandbox/cliclient_test.go index bfee25e57..a7136f899 100644 --- a/pkg/driver/microsandbox/cliclient_test.go +++ b/pkg/driver/microsandbox/cliclient_test.go @@ -5,6 +5,8 @@ import ( "strings" "testing" "time" + + "github.com/devsy-org/devsy/pkg/flags/names" ) func TestCreateArgsFull(t *testing.T) { @@ -27,7 +29,7 @@ func TestCreateArgsFull(t *testing.T) { args := createArgs(wsName, spec) // name comes first, image last. - if args[0] != cmdCreate || args[1] != flagName || args[2] != wsName { + if args[0] != names.Create || args[1] != names.Flag(names.Name) || args[2] != wsName { t.Errorf("prefix = %v", args[:3]) } if args[len(args)-1] != testImg { @@ -36,7 +38,7 @@ func TestCreateArgsFull(t *testing.T) { want := [][2]string{ {"--entrypoint", "/bin/sh -c sleep infinity"}, - {flagEnv, "K=v"}, + {names.Flag(names.Env), "K=v"}, {"--label", "devsy.sh/user=vscode"}, {"--idle-timeout", "1m30s"}, {"--memory", "2048M"}, @@ -59,7 +61,7 @@ func TestCreateArgsFull(t *testing.T) { func TestCreateArgsMinimal(t *testing.T) { args := createArgs(wsName, sandboxSpec{Image: testImg}) // Only name + image; no sizing/runtime flags for a bare spec. - want := []string{cmdCreate, flagName, wsName, testImg} + want := []string{names.Create, names.Flag(names.Name), wsName, testImg} if !slices.Equal(args, want) { t.Errorf("args = %v, want %v", args, want) } @@ -97,7 +99,15 @@ func TestResourceArgsOmitsZero(t *testing.T) { } func TestRedactArgsMasksEnvValues(t *testing.T) { - args := []string{cmdCreate, flagEnv, "TOKEN=s3cret", "--label", "k=v", flagEnv, "PLAIN=ok"} + args := []string{ + names.Create, + names.Flag(names.Env), + "TOKEN=s3cret", + "--label", + "k=v", + names.Flag(names.Env), + "PLAIN=ok", + } got := redactArgs(args) if strings.Contains(got, "s3cret") || strings.Contains(got, "ok") { t.Errorf("env values leaked: %q", got) From f29aa4177babc028c58b850996562f748390b533 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 26 Jul 2026 21:58:54 -0500 Subject: [PATCH 5/5] fix(gitsshsigning): preserve CRLF endings when removing signing helper #765's refactor widened the per-line trim from "\n" to "\r\n", so removing the SSH-signing section from a CRLF-authored gitconfig rewrote every line to LF. Trim only the LF to leave untouched lines' endings intact. --- pkg/gitsshsigning/helper.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/gitsshsigning/helper.go b/pkg/gitsshsigning/helper.go index ae57c0eb0..53fe1c3fe 100644 --- a/pkg/gitsshsigning/helper.go +++ b/pkg/gitsshsigning/helper.go @@ -176,7 +176,9 @@ type gitConfigFilter struct { func removeSignatureHelper(content string) string { f := &gitConfigFilter{current: sectionNone} for line := range strings.Lines(content) { - f.process(strings.TrimRight(line, "\r\n")) + // Trim only the LF so a CRLF file keeps its \r and is not rewritten to + // LF on the lines this filter passes through unchanged. + f.process(strings.TrimRight(line, "\n")) } f.flush()