cfergeau · cfergeau · May 26, 2025 · May 6, 2025 · May 6, 2025 · May 7, 2025
diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml
@@ -43,6 +43,14 @@ jobs:
           go-version-file: go.mod
       - name: Build
         run: make
+
+      - name: Install qemu-img
+        run: |
+          brew install qemu
+
+      - name: Verify qemu-img is installed
+        run: qemu-img --version
+
       - name: Test
         if: matrix.os != 'macOS-14'
         run: make test

diff --git a/OWNERS b/OWNERS
@@ -8,4 +8,5 @@ reviewers:
   - baude
   - cfergeau
   - lstocchi
+  - nirs
   - praveenkumar
diff --git a/README.md b/README.md
@@ -1,24 +1,11 @@
-vfkit - Simple command line tool to start VMs through the macOS Virtualization framework
+vfkit - Command-line tool to start VMs on macOS
 ====
 
 ### Introduction
 
 vfkit offers a command-line interface to start virtual machines using the [macOS Virtualization framework](https://developer.apple.com/documentation/virtualization).
 It also provides a `github.com/crc-org/vfkit/pkg/config` go package.
-This package provides a native Go API to generate the vfkit command line.
-
-
-### Installation
-
-vfkit is available in brew:
-
-```
-brew install vfkit
-```
-
-### Building
-
-From the root direction of this repository, run `make`.
+This package implements a native Go API to generate the vfkit command line.
 
 ### Usage
 
@@ -31,26 +18,21 @@ See https://github.com/crc-org/vfkit/blob/main/doc/usage.md
 - [Containers Plumbing 2023](https://crc.dev/blog/posts/2023-03-22-containers-plumbing/)
 - [FOSDEM 2023](https://fosdem.org/2023/schedule/event/govfkit/)
 
+### Adopters
 
-### Background
-
-The work in this repository makes use of https://github.com/Code-Hex/vz which provides go bindings for macOS virtualization framework.
-The lifetime of virtual machines created using the virtualization framework is tied to the filetime of the process where they were created.
-When using `Code-Hex/vz`, this means the virtual machine will be terminated at the end of the go process using these bindings.
-Spawning a `vfkit` process gives more flexibility and more control over the lifetime of the virtual machine.
+- [minikube](https://minikube.sigs.k8s.io/) 1.35.0 and newer - minikube quickly sets up a local Kubernetes cluster
+- [podman](https://podman.io/) 5.0 and newer - podman is a free software CLI tool to manage containers, pods and images
+- [crc](https://crc.dev/) - crc sets up local OpenShift or MicroShift clusters for development and testing purposes
+- [ovm](https://github.com/oomol-lab/ovm) - ovm is used by Oomol Studio to run linux containers on macOS
 
+### Installation
 
-The kernel must be uncompressed before use as no bootloader is used, as
-documented in https://www.kernel.org/doc/Documentation/arm64/booting.txt
+vfkit is available in brew:
 
 ```
-3. Decompress the kernel image
-------------------------------
+brew install vfkit
+```
 
-Requirement: OPTIONAL
+### Building
 
-The AArch64 kernel does not currently provide a decompressor and therefore
-requires decompression (gzip etc.) to be performed by the boot loader if a
-compressed Image target (e.g. Image.gz) is used.  For bootloaders that do not
-implement this requirement, the uncompressed Image target is available instead.
-```
+From the root direction of this repository, run `make`.
diff --git a/cmd/vfkit/main.go b/cmd/vfkit/main.go
@@ -136,8 +136,6 @@ func runVFKit(vmConfig *config.VirtualMachine, opts *cmdline.Options) error {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
-	util.SetupExitSignalHandling()
-
 	gpuDevs := vmConfig.VirtioGPUDevices()
 	if opts.UseGUI && len(gpuDevs) > 0 {
 		gpuDevs[0].UsesGUI = true
@@ -157,6 +155,26 @@ func runVFKit(vmConfig *config.VirtualMachine, opts *cmdline.Options) error {
 		}
 		srv.Start()
 	}
+
+	shutdownFunc := func() {
+		log.Debugf("shutting down...")
+		stopped, err := vfVM.RequestStop()
+		if err != nil {
+			log.Errorf("failed to shutdown VM: %v", err)
+		} else if !stopped {
+			log.Warnf("VM did not acknowledge stop request")
+		}
+		if err := waitForVMState(vfVM, vz.VirtualMachineStateStopped, time.After(5*time.Second)); err != nil {
+			log.Warnf("failed to wait for VM stop: %v, forcing stop", err)
+			if forceErr := vfVM.Stop(); forceErr != nil {
+				log.Errorf("failed to force stop VM: %v", forceErr)
+			}
+		} else {
+			log.Debugf("VM stopped gracefully")
+		}
+
+	}
+	util.SetupExitSignalHandling(shutdownFunc)
 	return runVirtualMachine(vmConfig, vfVM)
 }
 
@@ -289,8 +307,9 @@ func generateCloudInitImage(files []string) (string, error) {
 	}
 
 	configFiles := map[string]io.Reader{
-		"user-data": nil,
-		"meta-data": nil,
+		"user-data":      nil,
+		"meta-data":      nil,
+		"network-config": nil,
 	}
 
 	hasConfigFile := false
@@ -306,7 +325,9 @@ func generateCloudInitImage(files []string) (string, error) {
 
 		filename := filepath.Base(path)
 		if _, ok := configFiles[filename]; ok {
-			hasConfigFile = true
+			if filename == "user-data" || filename == "meta-data" {
+				hasConfigFile = true
+			}
 			configFiles[filename] = file
 		}
 	}

diff --git a/cmd/vfkit/main_test.go b/cmd/vfkit/main_test.go
@@ -70,6 +70,7 @@ func TestGenerateCloudInitImage(t *testing.T) {
 	iso, err := generateCloudInitImage([]string{
 		filepath.Join(assetsDir, "user-data"),
 		filepath.Join(assetsDir, "meta-data"),
+		filepath.Join(assetsDir, "network-config"),
 	})
 	require.NoError(t, err)
 

diff --git a/contrib/scripts/start-gvproxy.sh b/contrib/scripts/start-gvproxy.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+# SPDX-FileCopyrightText: The vfkit authors
+# SPDX-License-Identifier: Apache-2.0
+#
+# This script can be used to start a raw disk image with vfkit using gvproxy
+# for usermode networking.
+# The mac address must be 5a:94:ef:e4:0c:ee as this is the address expected by gvproxy.
+#
+# After the VM is running, its ssh port is reachable on port 2223 on localhost (127.0.0.1).
+#
+# If the path to `--listen-vfkit` is too long (more than ~100 characters), then
+# gvproxy/vfkit will fail to start as a unix socket filename must be less than
+# that.
+#
+# This script creates an overlay for the disk image, the disk image is not modified.
+#
+set -exuo pipefail
+
+: "${GVPROXY:=gvproxy}"
+: "${VFKIT:=./out/vfkit}"
+
+DISK_IMAGE="${1?Usage: $0 diskimage}"
+VM_NAME="$(basename ${DISK_IMAGE})"
+
+${GVPROXY} -mtu 1500 -ssh-port 2223 -listen-vfkit unixgram://$(pwd)/${VM_NAME}.sock  -log-file ${VM_NAME}.gvproxy.log --pid-file ${VM_NAME}.gvproxy.pid &
+
+TO_REMOVE="${VM_NAME}.sock ${VM_NAME}.gvproxy.pid ${VM_NAME}.overlay.img ${VM_NAME}.efistore.nvram"
+trap 'if [[ -f "${VM_NAME}.gvproxy.pid" ]]; then kill $(cat ${VM_NAME}.gvproxy.pid); fi; rm -f ${TO_REMOVE}' EXIT
+
+cp -c ${DISK_IMAGE} "${VM_NAME}".overlay.img
+
+${VFKIT} --cpus 2 --memory 2048 \
+    --bootloader efi,variable-store=${VM_NAME}.efistore.nvram,create \
+    --device virtio-blk,path=${VM_NAME}.overlay.img \
+    --device virtio-serial,logFilePath=${VM_NAME}.log \
+    --device virtio-net,unixSocketPath=$(pwd)/${VM_NAME}.sock,mac=5a:94:ef:e4:0c:ee \
+    --device virtio-rng
diff --git a/contrib/scripts/start-with-cloud-init.sh b/contrib/scripts/start-with-cloud-init.sh
@@ -12,20 +12,24 @@
 # The $DISK_IMG variable needs to be set by the user to a
 # valid image path for the VM.
 #
+# The script has optional support for network-config. The user
+# can specify a network-config file using $NETWORK_CONFIG variable.
+#
 # Once the VM is running, the user can connect to it using their
 # provided key. The VM IP can be found in `/var/db/dhcpd_leases`
 # by searching for the HOST_NAME or MAC address (72:20:43:d4:38:62).
 #
 # Example:
 # $ SSH_USER=test HOST_NAME=vm1 DISK_IMG=Fedora-Cloud-Base-AmazonEC2-41-1.4.aarch64.raw \
-# SSH_PUB_KEY=id_rsa.pub ./contrib/scripts/start-with-cloud-init.sh
+# SSH_PUB_KEY=id_rsa.pub NETWORK_CONFIG=network-config ./contrib/scripts/start-with-cloud-init.sh
 #
 # $ ssh -i id_rsa test@192.168.64.14
 
 set -exu
 
 HOST_NAME=${HOST_NAME:-"vfkit-vm"}
 SSH_USER=${SSH_USER:-"test"}
+NETWORK_CONFIG=${NETWORK_CONFIG:-}
 
 if [ ! -f "$SSH_PUB_KEY" ]; then
   echo "Error: '$SSH_PUB_KEY' does not exist"
@@ -37,6 +41,11 @@ if [ ! -f "$DISK_IMG" ]; then
   exit 1
 fi
 
+if [ -n "$NETWORK_CONFIG" ] && [ ! -f "$NETWORK_CONFIG" ]; then
+  echo "Error: '$NETWORK_CONFIG' does not exist"
+  exit 1
+fi
+
 PUBLIC_KEY=$(cat "$SSH_PUB_KEY")
 
 mkdir -p out
@@ -61,9 +70,14 @@ instance-id: $HOST_NAME
 local-hostname: $HOST_NAME
 EOF
 
+CLOUD_INIT_ARGS="out/user-data,out/meta-data"
+if [ -n "$NETWORK_CONFIG" ]; then
+  CLOUD_INIT_ARGS="$CLOUD_INIT_ARGS,$NETWORK_CONFIG"
+fi
+
 ./out/vfkit --cpus 2 --memory 2048 \
     --bootloader efi,variable-store=out/efistore.nvram,create \
-    --cloud-init out/user-data,out/meta-data \
+    --cloud-init "$CLOUD_INIT_ARGS" \
     --device virtio-blk,path="$DISK_IMG" \
     --device virtio-serial,logFilePath=out/cloud-init.log \
     --device virtio-net,nat,mac=72:20:43:d4:38:62 \

diff --git a/doc/usage.md b/doc/usage.md
@@ -1,7 +1,7 @@
 # vfkit Command Line
 
 The `vfkit` executable can be used to create a virtual machine (VM) using macOS virtualization framework.
-The virtual machine will be terminated as soon as the `vfkit` process exits.
+The virtual machine will be started when `vfkit` starts and will be terminated as soon as the `vfkit` process exits.
 Its configuration can be specified through command line options.
 
 Specifying VM bootloader configuration is mandatory.
@@ -55,6 +55,21 @@ A bootloader is required to tell vfkit _how_ it should start the guest OS.
 `--bootloader linux` replaces the legacy `--kernel`, `--kernel-cmdline` and `--initrd` options.
 It allows to specify which kernel and initrd should be used when starting the VM.
 
+On Apple Silicon hardware (M1 CPUs and newer), when using `--bootloader linux`, the kernel must be uncompressed before use as documented in https://www.kernel.org/doc/Documentation/arm64/booting.txt. `vfkit` will exit with an error if it detects a compressed kernel when running on Apple silicon. There are no such requirements when using `--bootloader efi`.
+
+Excerpt from the kernel’s `booting.txt`:
+```
+3. Decompress the kernel image
+------------------------------
+
+Requirement: OPTIONAL
+
+The AArch64 kernel does not currently provide a decompressor and therefore
+requires decompression (gzip etc.) to be performed by the boot loader if a
+compressed Image target (e.g. Image.gz) is used.  For bootloaders that do not
+implement this requirement, the uncompressed Image target is available instead.
+```
+
 #### Arguments
 
 - `kernel`: path to the kernel to use to start the virtual machine. The kernel *must* be uncompressed or the VM will hang when trying to start. See [the kernel documentation](https://www.kernel.org/doc/Documentation/arm64/booting.txt) for more details.
@@ -164,7 +179,7 @@ Vfkit can create this ISO image automatically, or you can provide a pre-made ISO
 
 ##### Automatic ISO Creation
 
-Vfkit allows you to pass the file paths of your `user-data` and `meta-data` files directly as arguments. 
+Vfkit allows you to pass the file paths of your `user-data`, `meta-data`, and `network-config` files directly as arguments.
 It will then handle the creation of the ISO image and the virtio-blk device internally.
 
 Example
@@ -273,6 +288,8 @@ This allows to connect to the export of the remote NBD server:
 
 The `--device virtio-net` option adds a network interface to the virtual machine. If it gets its IP address through DHCP, its IP can be found in `/var/db/dhcpd_leases` on the host.
 
+vfkit only supports NAT networking on its own. However, it integrates with [gvisor-tap-vsock](https://github.com/containers/gvisor-tap-vsock) for a user-mode networking stack, and [vmnet-helper](https://github.com/nirs/vmnet-helper) for shared/bridged/host networking through vmnet.
+
 #### Arguments
 - `mac`: optional argument to specify the MAC address of the VM. If it's omitted, a random MAC address will be used.
 - `fd`: file descriptor to attach to the guest network interface. The file descriptor must be a connected datagram socket. See [VZFileHandleNetworkDeviceAttachment](https://developer.apple.com/documentation/virtualization/vzfilehandlenetworkdeviceattachment?language=objc) for more details.
@@ -294,6 +311,9 @@ This adds a virtio-net device to the VM, and redirects all the network traffic o
 ```
 This is useful in combination with usermode networking stacks such as [gvisor-tap-vsock](https://github.com/containers/gvisor-tap-vsock).
 
+See [this shell script](https://github.com/nirs/vmnet-helper/blob/main/examples/vfkit.sh) for an example of networking using `vmnet-helper`.
+See [this shell script](https://github.com/crc-org/vfkit/blob/main/contrib/scripts/start-gvproxy.sh) for an example of networking using `gvproxy`.
+
 
 ### Serial Port
 

diff --git a/go.mod b/go.mod
@@ -5,11 +5,12 @@ go 1.23.0
 toolchain go1.23.6
 
 require (
-	github.com/Code-Hex/vz/v3 v3.6.0
+	github.com/Code-Hex/vz/v3 v3.7.0
 	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/containers/common v0.62.3
-	github.com/crc-org/crc/v2 v2.49.0
-	github.com/gin-gonic/gin v1.10.0
+	github.com/crc-org/crc/v2 v2.51.0
+	github.com/gin-gonic/gin v1.10.1
+	github.com/inetaf/tcpproxy v0.0.0-20250222171855-c4b9df066048
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/pkg/term v1.1.0
 	github.com/prashantgupta24/mac-sleep-notifier v1.0.1
@@ -18,10 +19,9 @@ require (
 	github.com/spf13/pflag v1.0.6
 	github.com/stretchr/testify v1.10.0
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8
-	golang.org/x/crypto v0.38.0
-	golang.org/x/mod v0.24.0
+	golang.org/x/crypto v0.39.0
+	golang.org/x/mod v0.25.0
 	golang.org/x/sys v0.33.0
-	inet.af/tcpproxy v0.0.0-20231102063150-2862066fc2a9
 )
 
 require (
@@ -35,7 +35,7 @@ require (
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/crc-org/machine v0.0.0-20240926103419-a943b47fd48b // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/ebitengine/purego v0.8.2 // indirect
+	github.com/ebitengine/purego v0.8.3 // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
@@ -61,16 +61,16 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/shirou/gopsutil/v4 v4.25.2 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.4 // indirect
 	github.com/tklauser/go-sysconf v0.3.14 // indirect
 	github.com/tklauser/numcpus v0.9.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	golang.org/x/arch v0.8.0 // indirect
-	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/net v0.40.0 // indirect
 	golang.org/x/term v0.32.0 // indirect
-	golang.org/x/text v0.25.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect