flatcar · robinschneider · Jul 15, 2026
diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml
@@ -0,0 +1,156 @@
+name: image
+
+on:
+  push:
+    branches:
+      - "**"
+    paths:
+      - .github/workflows/image.yaml
+      - Makefile
+      - Dockerfile
+      - nvidia-driver
+      - empty
+  release:
+    types:
+      - published
+  workflow_dispatch:
+    inputs:
+      driver_versions:
+        description: Space-separated NVIDIA driver versions to build (empty = use DRIVER_VERSIONS from the Makefile)
+        required: false
+        default: ""
+      flatcar_versions:
+        description: Space-separated Flatcar VERSION_ID values (GPU Operator tag suffixes and precompiled builds)
+        required: false
+        default: "4593.2.0 4593.2.1 4593.2.2 4593.2.3"
+      build_precompiled:
+        description: Also build precompiled driver images for each driver/Flatcar combination
+        type: boolean
+        required: false
+        default: true
+      release:
+        description: Tag images without a dev prefix (e.g. 580.167.08-flatcar4593.2.0) for use with plain driver.version values
+        type: boolean
+        required: false
+        default: false
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  CUDA_VERSION: 13.2.1
+  IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/driver
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.vars.outputs.version }}
+      driver_versions: ${{ steps.vars.outputs.driver_versions }}
+      flatcar_versions: ${{ steps.vars.outputs.flatcar_versions }}
+      flatcar_versions_plain: ${{ steps.vars.outputs.flatcar_versions_plain }}
+      build_precompiled: ${{ steps.vars.outputs.build_precompiled }}
+    steps:
+      - uses: actions/checkout@v6
+        name: Check out code
+
+      - name: Compute build matrix
+        id: vars
+        run: |
+          # Branch pushes build dev images with a unique commit-based prefix;
+          # GitHub Releases (or a manual dispatch with release=true) build clean tags
+          # (<driver_version>-flatcar<version_id>) matching the GPU Operator's default
+          # image reference scheme.
+          if [[ "${{ github.event_name }}" == "release" ]] || \
+             [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.release }}" == "true" ]]; then
+            echo "version=" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=dev-${GITHUB_SHA:0:8}" >> "$GITHUB_OUTPUT"
+          fi
+          # Default to the driver versions maintained in the Makefile
+          DRIVERS=$(sed -nE 's/^DRIVER_VERSIONS \?= *//p' Makefile)
+          FLATCARS="4593.2.0 4593.2.1 4593.2.2 4593.2.3"
+          PRECOMPILED="true"
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            if [[ -n "${{ inputs.driver_versions }}" ]]; then
+              DRIVERS="${{ inputs.driver_versions }}"
+            fi
+            FLATCARS="${{ inputs.flatcar_versions }}"
+            PRECOMPILED="${{ inputs.build_precompiled }}"
+          fi
+          echo "Driver versions: $DRIVERS"
+          echo "driver_versions=$(echo "$DRIVERS" | jq -R -c 'split(" ") | map(select(length > 0))')" >> "$GITHUB_OUTPUT"
+          echo "flatcar_versions=$(echo "$FLATCARS" | jq -R -c 'split(" ") | map(select(length > 0))')" >> "$GITHUB_OUTPUT"
+          echo "flatcar_versions_plain=$FLATCARS" >> "$GITHUB_OUTPUT"
+          echo "build_precompiled=$PRECOMPILED" >> "$GITHUB_OUTPUT"
+
+  flatcar:
+    needs: setup
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        driver_version: ${{ fromJSON(needs.setup.outputs.driver_versions) }}
+    env:
+      VERSION: ${{ needs.setup.outputs.version }}
+    steps:
+      - uses: actions/checkout@v6
+        name: Check out code
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Flatcar driver image
+        run: |
+          VERSION="$VERSION" \
+          PUSH_ON_BUILD=true \
+          make build-${{ matrix.driver_version }}
+
+      - name: Tag image for GPU Operator Flatcar suffix
+        run: |
+          BASE_TAG="${VERSION:+${VERSION}-}${{ matrix.driver_version }}-flatcar"
+          for FLATCAR_VERSION in ${{ needs.setup.outputs.flatcar_versions_plain }}; do
+            docker buildx imagetools create \
+              -t "${IMAGE_NAME}:${BASE_TAG}-flatcar${FLATCAR_VERSION}" \
+              "${IMAGE_NAME}:${BASE_TAG}"
+          done
+
+  precompiled-flatcar:
+    needs: setup
+    if: needs.setup.outputs.build_precompiled == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        driver_version: ${{ fromJSON(needs.setup.outputs.driver_versions) }}
+        flatcar_version: ${{ fromJSON(needs.setup.outputs.flatcar_versions) }}
+    env:
+      VERSION: ${{ needs.setup.outputs.version }}
+    steps:
+      - uses: actions/checkout@v6
+        name: Check out code
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push precompiled Flatcar driver image
+        run: |
+          VERSION="$VERSION" \
+          FLATCAR_VERSION="${{ matrix.flatcar_version }}" \
+          PUSH_ON_BUILD=true \
+          make build-precompiled-${{ matrix.driver_version }}
diff --git a/.idea/.gitignore b/.idea/.gitignore
diff --git a/.idea/gpu-driver-container.iml b/.idea/gpu-driver-container.iml
diff --git a/.idea/modules.xml b/.idea/modules.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,132 @@
+ARG CUDA_VERSION=13.2.1
+FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu24.04 AS build
+
+ENV NVIDIA_VISIBLE_DEVICES=void
+
+ARG TARGETARCH
+ENV TARGETARCH=$TARGETARCH
+
+SHELL ["/bin/bash", "-c"]
+
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main universe" > /etc/apt/sources.list && \
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates main universe" >> /etc/apt/sources.list && \
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-security main universe" >> /etc/apt/sources.list && \
+        usermod -o -u 0 -g 0 _apt ; \
+    elif [ "$TARGETARCH" = "arm64" ]; then \
+        echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports jammy main universe" > /etc/apt/sources.list && \
+        echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports jammy-updates main universe" >> /etc/apt/sources.list && \
+        echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports jammy-security main universe" >> /etc/apt/sources.list && \
+        usermod -o -u 0 -g 0 _apt ; \
+    else \
+        echo "TARGETARCH doesn't match a known arch target" ; \
+        exit 1 ; \
+    fi
+
+RUN if [ "$TARGETARCH" = "amd64" ]; then dpkg --add-architecture i386 ; fi && \
+    apt-get update && apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        apt-utils \
+        bc \
+        binutils \
+        build-essential \
+        ca-certificates \
+        curl \
+        gcc-14 \
+        gnupg2 \
+        jq \
+        kmod \
+        libelf-dev \
+        libssl-dev \
+        patchelf \
+        pahole \
+        file \
+        fdisk \
+        software-properties-common && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors -o /usr/local/bin/donkey https://github.com/3XX0/donkey/releases/download/v1.1.0/donkey && \
+    curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors -o /usr/local/bin/extract-vmlinux https://raw.githubusercontent.com/torvalds/linux/master/scripts/extract-vmlinux && \
+    chmod +x /usr/local/bin/donkey /usr/local/bin/extract-vmlinux
+
+ARG BASE_URL=https://us.download.nvidia.com/tesla
+ARG DRIVER_VERSION=580.95.05
+ENV DRIVER_VERSION=$DRIVER_VERSION
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install the userspace components and copy the kernel module sources.
+RUN cd /tmp && \
+    DRIVER_ARCH="$TARGETARCH" && DRIVER_ARCH=${DRIVER_ARCH/amd64/x86_64} && DRIVER_ARCH=${DRIVER_ARCH/arm64/aarch64} && \
+    curl -fSsl --retry 5 --retry-delay 5 --retry-all-errors -O $BASE_URL/$DRIVER_VERSION/NVIDIA-Linux-$DRIVER_ARCH-$DRIVER_VERSION.run && \
+    sh NVIDIA-Linux-${DRIVER_ARCH}-$DRIVER_VERSION.run -x && \
+    cd NVIDIA-Linux-${DRIVER_ARCH}-$DRIVER_VERSION*/ && \
+    ./nvidia-installer --silent \
+                       --no-kernel-module \
+                       --install-compat32-libs \
+                       --no-nouveau-check \
+                       --no-nvidia-modprobe \
+                       --no-rpms \
+                       --no-backup \
+                       --no-check-for-alternate-installs \
+                       --no-libglx-indirect \
+                       --no-install-libglvnd \
+                       --x-prefix=/tmp/null \
+                       --x-module-path=/tmp/null \
+                       --x-library-path=/tmp/null \
+                       --x-sysconfig-path=/tmp/null && \
+    mkdir -p /usr/src/nvidia-$DRIVER_VERSION && \
+    mv LICENSE mkprecompiled kernel /usr/src/nvidia-$DRIVER_VERSION && \
+    sed '9,${/^\(kernel\|LICENSE\)/!d}' .manifest > /usr/src/nvidia-$DRIVER_VERSION/.manifest && \
+    rm -rf /tmp/*
+
+# Install fabric manager and NSCQ for NVSwitch based systems (e.g. HGX/DGX A100, H100, H200).
+# Newer CUDA base images ship the cuda-keyring package (repo list is a dpkg conffile —
+# do NOT delete it, reinstalling the same version won't restore it). Older images ship
+# a bare cuda.list without signed-by, which conflicts with cuda-keyring; only in that
+# case remove it and install the keyring.
+# NVIDIA publishes fabric manager and NSCQ in lockstep with each datacenter driver
+# release (versions.mk only tracks such versions), so pin the exact driver version
+# and fail fast if it is missing. Packages are branch-suffixed below branch 580.
+RUN OS_ARCH=${TARGETARCH/amd64/x86_64} && OS_ARCH=${OS_ARCH/arm64/sbsa} && \
+    if ! dpkg -s cuda-keyring > /dev/null 2>&1; then \
+        rm -f /etc/apt/sources.list.d/cuda*.list && \
+        curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors -o /tmp/cuda-keyring.deb \
+            https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${OS_ARCH}/cuda-keyring_1.1-1_all.deb && \
+        dpkg -i /tmp/cuda-keyring.deb && rm -f /tmp/cuda-keyring.deb ; \
+    fi && \
+    apt-get update && \
+    DRIVER_BRANCH=${DRIVER_VERSION%%.*} && \
+    if [ "$DRIVER_BRANCH" -ge 580 ]; then \
+        FM_PKG=nvidia-fabricmanager NSCQ_PKG=libnvidia-nscq ; \
+    else \
+        FM_PKG=nvidia-fabricmanager-${DRIVER_BRANCH} NSCQ_PKG=libnvidia-nscq-${DRIVER_BRANCH} ; \
+    fi && \
+    apt-get install -y --no-install-recommends "${FM_PKG}=${DRIVER_VERSION}*" "${NSCQ_PKG}=${DRIVER_VERSION}*" && \
+    apt-mark hold "${FM_PKG}" "${NSCQ_PKG}" && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY nvidia-driver /usr/local/bin
+
+WORKDIR /usr/src/nvidia-$DRIVER_VERSION
+
+ARG PUBLIC_KEY=empty
+COPY ${PUBLIC_KEY} kernel/pubkey.x509
+
+# Add NGC DL license from the CUDA image
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+
+ENV CC=gcc-14
+
+# When FLATCAR_VERSION is set, build the kernel modules for that Flatcar release
+# at image build time and pack them as a precompiled package, so no compilation
+# happens at runtime. When unset, a generic image is built and the modules are
+# compiled at runtime for the running kernel.
+ARG FLATCAR_VERSION
+ENV FLATCAR_VERSION=$FLATCAR_VERSION
+RUN if [ -n "$FLATCAR_VERSION" ]; then \
+        CREATE_PRECOMPILED_PACKAGE=true nvidia-driver update -f "$FLATCAR_VERSION" ; \
+    fi
+
+ENTRYPOINT ["nvidia-driver", "init"]
diff --git a/Makefile b/Makefile
@@ -0,0 +1,51 @@
+DOCKER ?= docker
+
+# DRIVER_VERSIONS contains the latest version of each active datacenter branch.
+DRIVER_VERSIONS ?= 580.167.08 595.71.05
+CUDA_VERSION ?= 13.2.1
+
+REGISTRY ?= ghcr.io/flatcar
+IMAGE_NAME ?= $(REGISTRY)/driver
+
+# VERSION optionally prefixes the image tag (e.g. VERSION=dev-abc12345 for dev builds).
+# Release tags follow the GPU Operator scheme:
+#   generic:     <driver-version>-flatcar
+#   precompiled: <driver-version>-flatcar<flatcar-version>
+VERSION ?=
+TAG_PREFIX = $(if $(VERSION),$(VERSION)-)
+
+PLATFORM ?= linux/amd64
+PUSH_ON_BUILD ?= false
+DOCKER_BUILD_OPTIONS ?= --output=type=image,push=$(PUSH_ON_BUILD)
+
+BUILD_TARGETS := $(patsubst %,build-%,$(DRIVER_VERSIONS))
+PRECOMPILED_TARGETS := $(patsubst %,build-precompiled-%,$(DRIVER_VERSIONS))
+
+.PHONY: build build-precompiled $(BUILD_TARGETS) $(PRECOMPILED_TARGETS)
+
+# Generic images: kernel modules are compiled at runtime for the running kernel.
+build: $(BUILD_TARGETS)
+
+$(BUILD_TARGETS): build-%:
+	$(DOCKER) buildx build --pull \
+		$(DOCKER_BUILD_OPTIONS) \
+		--platform=$(PLATFORM) \
+		--tag $(IMAGE_NAME):$(TAG_PREFIX)$*-flatcar \
+		--build-arg DRIVER_VERSION="$*" \
+		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
+		$(CURDIR)
+
+# Precompiled images: kernel modules for a specific Flatcar release are built and
+# packed at image build time. Requires FLATCAR_VERSION (a Flatcar VERSION_ID).
+build-precompiled: $(PRECOMPILED_TARGETS)
+
+$(PRECOMPILED_TARGETS): build-precompiled-%:
+	@test -n "$(FLATCAR_VERSION)" || { echo "FLATCAR_VERSION must be set (e.g. FLATCAR_VERSION=4593.2.0)"; exit 1; }
+	$(DOCKER) buildx build --pull \
+		$(DOCKER_BUILD_OPTIONS) \
+		--platform=$(PLATFORM) \
+		--tag $(IMAGE_NAME):$(TAG_PREFIX)$*-flatcar$(FLATCAR_VERSION) \
+		--build-arg DRIVER_VERSION="$*" \
+		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
+		--build-arg FLATCAR_VERSION="$(FLATCAR_VERSION)" \
+		$(CURDIR)