feat: Add Fabric Manager partition support for NVLink-enabled multi-GPU VMs

Dockerfile

@@ -31,16 +31,25 @@ ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
 ARG GOLANG_VERSION=1.24.0
+ARG FM_VERSION=580
 
 RUN yum install -y wget tar gzip make gcc glibc-devel \
- && case "$TARGETARCH" in \
-      amd64) GO_TARBALL=go${GOLANG_VERSION}.linux-amd64.tar.gz ;; \
-      arm64) GO_TARBALL=go${GOLANG_VERSION}.linux-arm64.tar.gz ;; \
-      *) echo "Unsupported TARGETARCH=${TARGETARCH}" && exit 1 ;; \
+    && case "$TARGETARCH" in \
+    amd64) GO_TARBALL=go${GOLANG_VERSION}.linux-amd64.tar.gz ;; \
+    arm64) GO_TARBALL=go${GOLANG_VERSION}.linux-arm64.tar.gz ;; \
+    *) echo "Unsupported TARGETARCH=${TARGETARCH}" && exit 1 ;; \
     esac \
- && wget -nv -O /tmp/go.tgz https://go.dev/dl/${GO_TARBALL} \
- && tar -C /usr/local -xzf /tmp/go.tgz \
- && rm -f /tmp/go.tgz
+    && wget -nv -O /tmp/go.tgz https://go.dev/dl/${GO_TARBALL} \
+    && tar -C /usr/local -xzf /tmp/go.tgz \
+    && rm -f /tmp/go.tgz
+
+# Install Fabric Manager SDK for NVLink partition support
+# Setup NVIDIA network repository and install FM devel package
+RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo \
+    && dnf clean expire-cache \
+    && dnf module enable -y nvidia-driver:${FM_VERSION}-open/default \
+    && dnf install -y nvidia-fabricmanager-devel-${FM_VERSION} \
+    && dnf clean all
 
 ENV GOROOT=/usr/local/go
 ENV GOPATH=/go
@@ -68,9 +77,11 @@ LABEL release="N/A"
 LABEL summary="NVIDIA device plugin for KubeVirt"
 LABEL description="See summary"
 
+COPY --from=builder /usr/lib64/libnvfm* /usr/lib64/
 COPY --from=builder /workspace/nvidia-kubevirt-gpu-device-plugin /usr/bin/
 COPY --from=builder /workspace/utils/pci.ids /usr/pci.ids
+ENV LD_LIBRARY_PATH=/usr/lib64
 
 USER 0:0
 
-CMD ["nvidia-kubevirt-gpu-device-plugin"]
+ENTRYPOINT ["nvidia-kubevirt-gpu-device-plugin"]

manifests/nvidia-kubevirt-gpu-device-plugin.yaml

@@ -13,27 +13,38 @@ spec:
         name: nvidia-kubevirt-gpu-dp-ds
     spec:
       priorityClassName: system-node-critical
+      hostNetwork: true # Required for FM API access on localhost:6666
       tolerations:
-      # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode.
-      # This, along with the annotation above marks this pod as a critical add-on.
-      - key: CriticalAddonsOnly
-        operator: Exists
+        - operator: "Exists"
+      nodeSelector:
+        nvidia.com/gpu.present: "true"
       containers:
-      - name: nvidia-kubevirt-gpu-dp-ctr
-        image: nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.4.0
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop: ["ALL"]
-        volumeMounts:
-          - name: device-plugin
-            mountPath: /var/lib/kubelet/device-plugins
-          - name: vfio
-            mountPath: /dev/vfio
+        - name: nvidia-kubevirt-gpu-dp-ctr
+          image: docker.io/anurlan/kubevirt-gpu-device-plugin:v1.5.0-fm
+          imagePullPolicy: Always
+          args:
+            - "--fm-enabled=true"
+            - "--fm-address=127.0.0.1:6666"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: device-plugin
+              mountPath: /var/lib/kubelet/device-plugins
+            - name: vfio
+              mountPath: /dev/vfio
+            - name: sys
+              mountPath: /sys
+              readOnly: true
       volumes:
         - name: device-plugin
           hostPath:
             path: /var/lib/kubelet/device-plugins
         - name: vfio
           hostPath:
             path: /dev/vfio
+        - name: sys
+          hostPath:
+            path: /sys
+            type: Directory

pkg/device_plugin/device_plugin.go

@@ -30,16 +30,20 @@ package device_plugin
 
 import (
 	"bufio"
+	"flag"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 
 	klog "k8s.io/klog/v2"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+
+	fm "kubevirt-gpu-device-plugin/pkg/fabric_manager"
 )
 
 const (
@@ -86,15 +90,63 @@ var readGpuIDForVgpu = readGpuIDForVgpuFunc
 var startVgpuDevicePlugin = startVgpuDevicePluginFunc
 var stop = make(chan struct{})
 
+// FM (Fabric Manager) configuration flags
+var (
+	fmEnabled = flag.Bool("fm-enabled", false, "Enable Fabric Manager partition support for NVLink")
+	fmAddress = flag.String("fm-address", "127.0.0.1:6666", "Fabric Manager daemon address")
+)
+
+// Global partition manager instance (shared across all device plugins)
+var partitionManager *fm.PartitionManager
+
+// InitiateDevicePlugin initiates the device plugin
 func InitiateDevicePlugin() {
+	// Parse command line flags
+	flag.Parse()
+
 	//Identifies GPUs and represents it in appropriate structures
 	createIommuDeviceMap()
 	//Identifies vGPUs and represents it in appropriate structures
 	createVgpuIDMap()
+
+	// Collect all discovered GPU BDFs for FM mapping
+	var allDevs []string
+	for _, devs := range deviceMap {
+		for _, d := range devs {
+			allDevs = append(allDevs, d.addr)
+		}
+	}
+	sort.Strings(allDevs)
+
+	// Initialize FM partition manager if enabled
+	if *fmEnabled {
+		log.Printf("Fabric Manager support ENABLED, connecting to %s", *fmAddress)
+		initFabricManager(allDevs)
+	} else {
+		log.Println("Fabric Manager support DISABLED")
+	}
+
 	//Creates and starts device plugin
 	createDevicePlugins()
 }
 
+// initFabricManager initializes the Fabric Manager partition manager
+func initFabricManager(devs []string) {
+	// Create native FM client and partition manager
+	fmClient := fm.NewNativeFMClient()
+	partitionManager = fm.NewPartitionManager(fmClient)
+
+	// Initialize: connects to FM, discovers partitions, builds tree
+	if err := partitionManager.Initialize(*fmAddress, devs); err != nil {
+		log.Printf("WARNING: Failed to initialize FM partition manager: %v", err)
+		log.Println("FM partition-aware allocation will be disabled")
+		partitionManager = nil
+		return
+	}
+
+	log.Printf("FM partition manager initialized successfully")
+}
+
 // Starts gpu pass through and vGPU device plugin
 func createDevicePlugins() {
 	var devicePlugins []*GenericDevicePlugin
@@ -128,6 +180,13 @@ func createDevicePlugins() {
 		}
 		log.Printf("DP Name %s", deviceName)
 		dp := NewGenericDevicePlugin(deviceName, "/dev/vfio/", devs)
+		// Set partition manager if FM is enabled
+		if partitionManager != nil {
+			dp.SetPartitionManager(partitionManager)
+			// Start reconciler if not already running (only need one for all device plugins)
+			resourceName := "nvidia.com/" + deviceName
+			partitionManager.StartReconciler(resourceName)
+		}
 		err := startDevicePlugin(dp)
 		if err != nil {
 			log.Printf("Error starting %s device plugin: %v", dp.deviceName, err)