From bd2aa17f34c9af6a350a21331154d2e766a444f5 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Wed, 15 Jul 2026 14:19:36 +0000 Subject: [PATCH] build gpu-driver-container Signed-off-by: Robin Schneider --- .github/workflows/image.yaml | 156 ++++++++++ .idea/.gitignore | 10 + .idea/gpu-driver-container.iml | 9 + .idea/modules.xml | 8 + .idea/vcs.xml | 7 + Dockerfile | 132 ++++++++ Makefile | 51 +++ README.md | 388 +++++++++++++++++++++++ build.sh | 10 + empty | 0 nvidia-driver | 552 +++++++++++++++++++++++++++++++++ run.sh | 14 + save.sh | 10 + update.sh | 13 + 14 files changed, 1360 insertions(+) create mode 100644 .github/workflows/image.yaml create mode 100644 .idea/.gitignore create mode 100644 .idea/gpu-driver-container.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100755 build.sh create mode 100644 empty create mode 100755 nvidia-driver create mode 100755 run.sh create mode 100755 save.sh create mode 100755 update.sh diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml new file mode 100644 index 0000000..34d4ae8 --- /dev/null +++ b/.github/workflows/image.yaml @@ -0,0 +1,156 @@ +name: image + +on: + push: + branches: + - "**" + paths: + - .github/workflows/image.yaml + - Makefile + - Dockerfile + - nvidia-driver + - empty + release: + types: + - published + workflow_dispatch: + inputs: + driver_versions: + description: Space-separated NVIDIA driver versions to build (empty = use DRIVER_VERSIONS from the Makefile) + required: false + default: "" + flatcar_versions: + description: Space-separated Flatcar VERSION_ID values (GPU Operator tag suffixes and precompiled builds) + required: false + default: "4593.2.0 4593.2.1 4593.2.2 4593.2.3" + build_precompiled: + description: Also build precompiled driver images for each driver/Flatcar combination + type: boolean + required: false + default: true + release: + description: Tag images without a dev prefix (e.g. 580.167.08-flatcar4593.2.0) for use with plain driver.version values + type: boolean + required: false + default: false + +permissions: + contents: read + packages: write + +env: + CUDA_VERSION: 13.2.1 + IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/driver + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.vars.outputs.version }} + driver_versions: ${{ steps.vars.outputs.driver_versions }} + flatcar_versions: ${{ steps.vars.outputs.flatcar_versions }} + flatcar_versions_plain: ${{ steps.vars.outputs.flatcar_versions_plain }} + build_precompiled: ${{ steps.vars.outputs.build_precompiled }} + steps: + - uses: actions/checkout@v6 + name: Check out code + + - name: Compute build matrix + id: vars + run: | + # Branch pushes build dev images with a unique commit-based prefix; + # GitHub Releases (or a manual dispatch with release=true) build clean tags + # (-flatcar) matching the GPU Operator's default + # image reference scheme. + if [[ "${{ github.event_name }}" == "release" ]] || \ + [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.release }}" == "true" ]]; then + echo "version=" >> "$GITHUB_OUTPUT" + else + echo "version=dev-${GITHUB_SHA:0:8}" >> "$GITHUB_OUTPUT" + fi + # Default to the driver versions maintained in the Makefile + DRIVERS=$(sed -nE 's/^DRIVER_VERSIONS \?= *//p' Makefile) + FLATCARS="4593.2.0 4593.2.1 4593.2.2 4593.2.3" + PRECOMPILED="true" + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + if [[ -n "${{ inputs.driver_versions }}" ]]; then + DRIVERS="${{ inputs.driver_versions }}" + fi + FLATCARS="${{ inputs.flatcar_versions }}" + PRECOMPILED="${{ inputs.build_precompiled }}" + fi + echo "Driver versions: $DRIVERS" + echo "driver_versions=$(echo "$DRIVERS" | jq -R -c 'split(" ") | map(select(length > 0))')" >> "$GITHUB_OUTPUT" + echo "flatcar_versions=$(echo "$FLATCARS" | jq -R -c 'split(" ") | map(select(length > 0))')" >> "$GITHUB_OUTPUT" + echo "flatcar_versions_plain=$FLATCARS" >> "$GITHUB_OUTPUT" + echo "build_precompiled=$PRECOMPILED" >> "$GITHUB_OUTPUT" + + flatcar: + needs: setup + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + driver_version: ${{ fromJSON(needs.setup.outputs.driver_versions) }} + env: + VERSION: ${{ needs.setup.outputs.version }} + steps: + - uses: actions/checkout@v6 + name: Check out code + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Flatcar driver image + run: | + VERSION="$VERSION" \ + PUSH_ON_BUILD=true \ + make build-${{ matrix.driver_version }} + + - name: Tag image for GPU Operator Flatcar suffix + run: | + BASE_TAG="${VERSION:+${VERSION}-}${{ matrix.driver_version }}-flatcar" + for FLATCAR_VERSION in ${{ needs.setup.outputs.flatcar_versions_plain }}; do + docker buildx imagetools create \ + -t "${IMAGE_NAME}:${BASE_TAG}-flatcar${FLATCAR_VERSION}" \ + "${IMAGE_NAME}:${BASE_TAG}" + done + + precompiled-flatcar: + needs: setup + if: needs.setup.outputs.build_precompiled == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + driver_version: ${{ fromJSON(needs.setup.outputs.driver_versions) }} + flatcar_version: ${{ fromJSON(needs.setup.outputs.flatcar_versions) }} + env: + VERSION: ${{ needs.setup.outputs.version }} + steps: + - uses: actions/checkout@v6 + name: Check out code + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push precompiled Flatcar driver image + run: | + VERSION="$VERSION" \ + FLATCAR_VERSION="${{ matrix.flatcar_version }}" \ + PUSH_ON_BUILD=true \ + make build-precompiled-${{ matrix.driver_version }} diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..30cf57e --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/gpu-driver-container.iml b/.idea/gpu-driver-container.iml new file mode 100644 index 0000000..5e764c4 --- /dev/null +++ b/.idea/gpu-driver-container.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..f5c478c --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..8306744 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6a1ab2c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,132 @@ +ARG CUDA_VERSION=13.2.1 +FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu24.04 AS build + +ENV NVIDIA_VISIBLE_DEVICES=void + +ARG TARGETARCH +ENV TARGETARCH=$TARGETARCH + +SHELL ["/bin/bash", "-c"] + +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main universe" > /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates main universe" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-security main universe" >> /etc/apt/sources.list && \ + usermod -o -u 0 -g 0 _apt ; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports jammy main universe" > /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports jammy-updates main universe" >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports jammy-security main universe" >> /etc/apt/sources.list && \ + usermod -o -u 0 -g 0 _apt ; \ + else \ + echo "TARGETARCH doesn't match a known arch target" ; \ + exit 1 ; \ + fi + +RUN if [ "$TARGETARCH" = "amd64" ]; then dpkg --add-architecture i386 ; fi && \ + apt-get update && apt-get install -y --no-install-recommends \ + apt-transport-https \ + apt-utils \ + bc \ + binutils \ + build-essential \ + ca-certificates \ + curl \ + gcc-14 \ + gnupg2 \ + jq \ + kmod \ + libelf-dev \ + libssl-dev \ + patchelf \ + pahole \ + file \ + fdisk \ + software-properties-common && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors -o /usr/local/bin/donkey https://github.com/3XX0/donkey/releases/download/v1.1.0/donkey && \ + curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors -o /usr/local/bin/extract-vmlinux https://raw.githubusercontent.com/torvalds/linux/master/scripts/extract-vmlinux && \ + chmod +x /usr/local/bin/donkey /usr/local/bin/extract-vmlinux + +ARG BASE_URL=https://us.download.nvidia.com/tesla +ARG DRIVER_VERSION=580.95.05 +ENV DRIVER_VERSION=$DRIVER_VERSION +ENV DEBIAN_FRONTEND=noninteractive + +# Install the userspace components and copy the kernel module sources. +RUN cd /tmp && \ + DRIVER_ARCH="$TARGETARCH" && DRIVER_ARCH=${DRIVER_ARCH/amd64/x86_64} && DRIVER_ARCH=${DRIVER_ARCH/arm64/aarch64} && \ + curl -fSsl --retry 5 --retry-delay 5 --retry-all-errors -O $BASE_URL/$DRIVER_VERSION/NVIDIA-Linux-$DRIVER_ARCH-$DRIVER_VERSION.run && \ + sh NVIDIA-Linux-${DRIVER_ARCH}-$DRIVER_VERSION.run -x && \ + cd NVIDIA-Linux-${DRIVER_ARCH}-$DRIVER_VERSION*/ && \ + ./nvidia-installer --silent \ + --no-kernel-module \ + --install-compat32-libs \ + --no-nouveau-check \ + --no-nvidia-modprobe \ + --no-rpms \ + --no-backup \ + --no-check-for-alternate-installs \ + --no-libglx-indirect \ + --no-install-libglvnd \ + --x-prefix=/tmp/null \ + --x-module-path=/tmp/null \ + --x-library-path=/tmp/null \ + --x-sysconfig-path=/tmp/null && \ + mkdir -p /usr/src/nvidia-$DRIVER_VERSION && \ + mv LICENSE mkprecompiled kernel /usr/src/nvidia-$DRIVER_VERSION && \ + sed '9,${/^\(kernel\|LICENSE\)/!d}' .manifest > /usr/src/nvidia-$DRIVER_VERSION/.manifest && \ + rm -rf /tmp/* + +# Install fabric manager and NSCQ for NVSwitch based systems (e.g. HGX/DGX A100, H100, H200). +# Newer CUDA base images ship the cuda-keyring package (repo list is a dpkg conffile — +# do NOT delete it, reinstalling the same version won't restore it). Older images ship +# a bare cuda.list without signed-by, which conflicts with cuda-keyring; only in that +# case remove it and install the keyring. +# NVIDIA publishes fabric manager and NSCQ in lockstep with each datacenter driver +# release (versions.mk only tracks such versions), so pin the exact driver version +# and fail fast if it is missing. Packages are branch-suffixed below branch 580. +RUN OS_ARCH=${TARGETARCH/amd64/x86_64} && OS_ARCH=${OS_ARCH/arm64/sbsa} && \ + if ! dpkg -s cuda-keyring > /dev/null 2>&1; then \ + rm -f /etc/apt/sources.list.d/cuda*.list && \ + curl -fsSL --retry 5 --retry-delay 5 --retry-all-errors -o /tmp/cuda-keyring.deb \ + https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${OS_ARCH}/cuda-keyring_1.1-1_all.deb && \ + dpkg -i /tmp/cuda-keyring.deb && rm -f /tmp/cuda-keyring.deb ; \ + fi && \ + apt-get update && \ + DRIVER_BRANCH=${DRIVER_VERSION%%.*} && \ + if [ "$DRIVER_BRANCH" -ge 580 ]; then \ + FM_PKG=nvidia-fabricmanager NSCQ_PKG=libnvidia-nscq ; \ + else \ + FM_PKG=nvidia-fabricmanager-${DRIVER_BRANCH} NSCQ_PKG=libnvidia-nscq-${DRIVER_BRANCH} ; \ + fi && \ + apt-get install -y --no-install-recommends "${FM_PKG}=${DRIVER_VERSION}*" "${NSCQ_PKG}=${DRIVER_VERSION}*" && \ + apt-mark hold "${FM_PKG}" "${NSCQ_PKG}" && \ + rm -rf /var/lib/apt/lists/* + +COPY nvidia-driver /usr/local/bin + +WORKDIR /usr/src/nvidia-$DRIVER_VERSION + +ARG PUBLIC_KEY=empty +COPY ${PUBLIC_KEY} kernel/pubkey.x509 + +# Add NGC DL license from the CUDA image +RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE + +ENV CC=gcc-14 + +# When FLATCAR_VERSION is set, build the kernel modules for that Flatcar release +# at image build time and pack them as a precompiled package, so no compilation +# happens at runtime. When unset, a generic image is built and the modules are +# compiled at runtime for the running kernel. +ARG FLATCAR_VERSION +ENV FLATCAR_VERSION=$FLATCAR_VERSION +RUN if [ -n "$FLATCAR_VERSION" ]; then \ + CREATE_PRECOMPILED_PACKAGE=true nvidia-driver update -f "$FLATCAR_VERSION" ; \ + fi + +ENTRYPOINT ["nvidia-driver", "init"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6c01984 --- /dev/null +++ b/Makefile @@ -0,0 +1,51 @@ +DOCKER ?= docker + +# DRIVER_VERSIONS contains the latest version of each active datacenter branch. +DRIVER_VERSIONS ?= 580.167.08 595.71.05 +CUDA_VERSION ?= 13.2.1 + +REGISTRY ?= ghcr.io/flatcar +IMAGE_NAME ?= $(REGISTRY)/driver + +# VERSION optionally prefixes the image tag (e.g. VERSION=dev-abc12345 for dev builds). +# Release tags follow the GPU Operator scheme: +# generic: -flatcar +# precompiled: -flatcar +VERSION ?= +TAG_PREFIX = $(if $(VERSION),$(VERSION)-) + +PLATFORM ?= linux/amd64 +PUSH_ON_BUILD ?= false +DOCKER_BUILD_OPTIONS ?= --output=type=image,push=$(PUSH_ON_BUILD) + +BUILD_TARGETS := $(patsubst %,build-%,$(DRIVER_VERSIONS)) +PRECOMPILED_TARGETS := $(patsubst %,build-precompiled-%,$(DRIVER_VERSIONS)) + +.PHONY: build build-precompiled $(BUILD_TARGETS) $(PRECOMPILED_TARGETS) + +# Generic images: kernel modules are compiled at runtime for the running kernel. +build: $(BUILD_TARGETS) + +$(BUILD_TARGETS): build-%: + $(DOCKER) buildx build --pull \ + $(DOCKER_BUILD_OPTIONS) \ + --platform=$(PLATFORM) \ + --tag $(IMAGE_NAME):$(TAG_PREFIX)$*-flatcar \ + --build-arg DRIVER_VERSION="$*" \ + --build-arg CUDA_VERSION="$(CUDA_VERSION)" \ + $(CURDIR) + +# Precompiled images: kernel modules for a specific Flatcar release are built and +# packed at image build time. Requires FLATCAR_VERSION (a Flatcar VERSION_ID). +build-precompiled: $(PRECOMPILED_TARGETS) + +$(PRECOMPILED_TARGETS): build-precompiled-%: + @test -n "$(FLATCAR_VERSION)" || { echo "FLATCAR_VERSION must be set (e.g. FLATCAR_VERSION=4593.2.0)"; exit 1; } + $(DOCKER) buildx build --pull \ + $(DOCKER_BUILD_OPTIONS) \ + --platform=$(PLATFORM) \ + --tag $(IMAGE_NAME):$(TAG_PREFIX)$*-flatcar$(FLATCAR_VERSION) \ + --build-arg DRIVER_VERSION="$*" \ + --build-arg CUDA_VERSION="$(CUDA_VERSION)" \ + --build-arg FLATCAR_VERSION="$(FLATCAR_VERSION)" \ + $(CURDIR) diff --git a/README.md b/README.md new file mode 100644 index 0000000..753acef --- /dev/null +++ b/README.md @@ -0,0 +1,388 @@ +# NVIDIA Drivers and Flatcar Container Linux + +[Flatcar Container Linux](https://kinvolk.io/flatcar-container-linux/) is a Linux distribution designed for container workloads. +To provide better security, Flatcar is immutable and contains minimal tools to run container workloads. This repository contains code to be able to build and provision NVIDIA drivers for Flatcar Linux through the use of containers. The NVIDIA driver container takes care of building the NVIDIA kernel modules and optionally loading them for running CUDA workloads. Together with the [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker), also provisioned using a container, you can start running GPU accelerated containers on Flatcar Linux. + +## Prerequisites + +Since the driver container uses a `loop` device to mount the Flatcar development environment, the `loop` kernel module should be loaded. +The NVIDIA driver also has a dependency on the following modules: `i2c_core` and `ipmi_msghandler`. For convenience, these modules can be +configured to be loaded at reboot. + +```bash +$ sudo modprobe -a loop i2c_core ipmi_msghandler \ + && echo -e "loop\ni2c_core\nipmi_msghandler" | sudo tee /etc/modules-load.d/driver.conf +``` + +If you're getting started with Flatcar on EC2, then Flatcar images for various regions can be found here: +https://kinvolk.io/docs/flatcar-container-linux/latest/installing/cloud/aws-ec2/ + +## Supported Drivers + +The following NVIDIA datacenter drivers have been tested with this container on Flatcar (kernel 6.12+): +1. `580.159.04` +1. `595.71.05` + +Other datacenter driver versions from the [NVIDIA driver downloads](https://www.nvidia.com/en-us/drivers/) are expected to work as +long as they support the running kernel. + +NVIDIA datacenter GPUs based on Pascal+ architecture (e.g. P100, V100, T4, A100, L40S, H100) are supported. On NVSwitch based +systems (e.g. HGX/DGX A100, H100, H200) the driver container ships the NVIDIA Fabric Manager and NSCQ library and starts the +Fabric Manager automatically when NVSwitch devices are detected (`/proc/driver/nvidia-nvswitch/devices`); this has been verified +on HGX H100 8-GPU systems. Note that NVLink5+ systems (e.g. GB200 NVL) additionally require the NVLink Subnet Manager, which is +not yet included. + +## Getting Started + +Setup the NVIDIA Container Toolkit using the [container-config](https://gitlab.com/nvidia/container-toolkit/container-config) project: +```bash +$ docker run --rm --privileged \ + -v "/etc/docker:/etc/docker" \ + -v "/run/nvidia:/run/nvidia" \ + -v "/run/docker.sock:/run/docker.sock" \ + -v "/opt/nvidia-runtime:/opt/nvidia-runtime" \ + -e "RUNTIME=docker" \ + -e "RUNTIME_ARGS=--socket /run/docker.sock" \ + -e "DOCKER_SOCKET=/run/docker.sock" \ + nvcr.io/nvidia/k8s/container-toolkit:1.4.7-ubuntu18.04 \ + "/opt/nvidia-runtime" +``` +You should see an output as shown below: +```console +time="2021-04-06T06:54:23Z" level=info msg="Setting up runtime" +time="2021-04-06T06:54:23Z" level=info msg="Starting 'setup' for docker" +time="2021-04-06T06:54:23Z" level=info msg="Parsing arguments: [/opt/nvidia-runtime/toolkit]" +time="2021-04-06T06:54:23Z" level=info msg="Successfully parsed arguments" +time="2021-04-06T06:54:23Z" level=info msg="Loading config: /etc/docker/daemon.json" +time="2021-04-06T06:54:23Z" level=info msg="Config file does not exist, creating new one" +time="2021-04-06T06:54:23Z" level=info msg="Flushing config" +time="2021-04-06T06:54:23Z" level=info msg="Successfully flushed config" +time="2021-04-06T06:54:23Z" level=info msg="Sending SIGHUP signal to docker" +time="2021-04-06T06:54:23Z" level=info msg="Signal received, exiting early" +time="2021-04-06T06:54:23Z" level=info msg="Shutting Down" +``` + +Restart `docker` to ensure that `nvidia` is added as a custom runtime: +```bash +$ sudo systemctl restart docker +$ docker info | grep -i nvidia + Runtimes: nvidia runc + Default Runtime: nvidia +``` + +The `container-config` would have also created a `daemon.json` for Docker in `/etc/docker/daemon.json` with the `nvidia` runtime binaries being located in the directory specified (`/opt/nvidia-runtime` in our example). + +## Workflow + +The driver container includes logic to precompile the NVIDIA kernel module interfaces and package them for later use. This has a +few advantages - reduced startup time since the module interfaces are already built; ability to build on systems without GPUs and +reduced image footprint as we no longer need to either download or ship the development environment within the image. +Note that the precompilation logic works when there is a matching running kernel and falls back to building the modules. + +To facilitate this optimization, the workflow can be: +1. Build the driver container +1. Commit and tag the image with a changed entrypoint +1. Run the driver container with the tagged image in the previous step + +Note that you can skip these steps on systems where you want to build and run the container directly by providing +the `init` argument to the driver container (instead of `update`) in the workflow below. By doing so, the driver +container will first build and then load the kernel modules. + +### Building the Driver Container + +Clone this repository and build a driver container image using the following as an example: + +```bash +$ git clone https://github.com/flatcar/gpu-driver-container.git \ + && cd gpu-driver-container +``` + +```bash +$ DRIVER_VERSION=460.32.03 +$ docker build --pull \ + --build-arg DRIVER_VERSION=$DRIVER_VERSION \ + --tag nvidia/nvidia-driver-flatcar:${DRIVER_VERSION} \ + --file Dockerfile . +``` + +### Precompile Kernel Interfaces + +Launch the driver container using the following as an example: + +```bash +$ docker run -d --privileged --pid=host \ + -v /run/nvidia:/run/nvidia:shared \ + -v /tmp/nvidia:/var/log \ + -v /usr/lib64/modules:/usr/lib64/modules \ + --name nvidia-driver \ + nvidia/nvidia-driver-flatcar:${DRIVER_VERSION} update +``` + +The building of the kernel modules takes a few minutes. You can also stream the logs of the container: + +```bash +$ docker logs -f nvidia-driver +``` +Once the modules are built, you should see an output as shown: + +```console +Compiling NVIDIA driver kernel modules with gcc (Gentoo Hardened 9.3.0-r1 p3) 9.3.0... +scripts/Makefile.lib:8: 'always' is deprecated. Please use 'always-y' instead +Cleaning up the development environment... +Checking NVIDIA driver packages... +Building NVIDIA driver package nvidia-modules-5.10.25... +Packaged precompiled driver into /usr/src/nvidia-460.32.03/kernel/precompiled/5.10.25-flatcar +Done +``` + +### Commit Image + +Now we can commit the image with a changed entrypoint. The image may also be stored in an image registry for use later. + +```bash +$ docker commit \ + --change='ENTRYPOINT ["nvidia-driver", "init"]' \ + nvidia-driver nvidia/nvidia-kmods-driver-flatcar:${DRIVER_VERSION} +``` + +### Running the Driver Container + +Run the driver container with the tagged image from the previous step: + +```bash +$ docker run -d --privileged --pid=host \ + -v /run/nvidia:/run/nvidia:shared \ + -v /tmp/nvidia:/var/log \ + -v /usr/lib64/modules:/usr/lib64/modules \ + nvidia/nvidia-kmods-driver-flatcar:${DRIVER_VERSION} +``` + +The driver container will look for the precompiled interfaces and then use those if available. +The startup time of the container in this case would be reduced: + +```console +... +========== NVIDIA Software Installer ========== +Starting installation of NVIDIA driver version 460.32.03 for Linux kernel version 5.10.25-flatcar +Stopping NVIDIA persistence daemon... +Unloading NVIDIA driver kernel modules... +Unmounting NVIDIA driver rootfs... +Checking NVIDIA driver packages... +Found NVIDIA driver package nvidia-modules-5.10.25 +Installing NVIDIA driver kernel modules... +Re-linking NVIDIA driver kernel modules... +depmod: WARNING: could not open /opt/nvidia/460.32.03/lib/modules/5.10.25-flatcar/modules.builtin: No such file or directory +Loading NVIDIA driver kernel modules... +Starting NVIDIA persistence daemon... +Mounting NVIDIA driver rootfs... +Done, now waiting for signal +``` + +You can also verify that the NVIDIA kernel modules are loaded in the system: + +```bash +$ lsmod | grep -i nvidia +nvidia_modeset 1228800 0 +nvidia_uvm 1130496 0 +nvidia 34078720 17 nvidia_uvm,nvidia_modeset +i2c_core 81920 3 nvidia,psmouse,i2c_piix4 +``` + +## Deploying the Precompiled Driver with the GPU Operator + +Instead of the manual Docker workflow above, a precompiled driver image can be built +with the `build-precompiled-` Make target (e.g. +`FLATCAR_VERSION=4593.2.0 make build-precompiled-580.167.08`) and deployed to a Kubernetes +cluster using the [NVIDIA GPU Operator](https://github.com/NVIDIA/gpu-operator) and Helm. + +### Prerequisites + +- A Kubernetes cluster with Flatcar Container Linux GPU nodes. The node's Flatcar + `VERSION_ID` and kernel must match the precompiled image (e.g. `4593.2.0` / + `6.12.81-flatcar`). +- `helm` v3 and `kubectl` with cluster-admin access. +- A precompiled driver image published to a registry, e.g. + `ghcr.io/robinschneider/driver:flatcar-test-5b98d7b9-595.71.05-flatcar4593.2.0`. + +### Image tag convention + +The GPU Operator composes the driver image reference as: + +``` +/:- +``` + +where `osTag` is derived from the node's OS labels (`flatcar` + `VERSION_ID`, e.g. +`flatcar4593.2.0`). For the image above, set +`driver.version=flatcar-test-5b98d7b9-595.71.05` and the operator appends +`-flatcar4593.2.0` automatically. + +### Installation + +```bash +helm repo add nvidia https://helm.ngc.nvidia.com/nvidia +helm repo update + +helm install gpu-operator nvidia/gpu-operator \ + --namespace gpu-operator --create-namespace \ + --set driver.repository=ghcr.io/robinschneider \ + --set driver.image=driver \ + --set driver.version=flatcar-test-5b98d7b9-595.71.05 \ + --set toolkit.installDir=/opt/nvidia-runtime +``` + +Notes: + +- `toolkit.installDir=/opt/nvidia-runtime` is **required on Flatcar**: the default + install path `/usr/local/nvidia` is on a read-only filesystem and the container + toolkit pods will fail with `CreateContainerError` otherwise. +- If the image is in a private registry, create a pull secret in the `gpu-operator` + namespace and add `--set driver.imagePullSecrets[0]=`. + +### Verification + +1. Wait for the driver pods and check that the precompiled package is used (no runtime + compilation): + + ```bash + kubectl -n gpu-operator logs -l app=nvidia-driver-daemonset | grep -E "Found NVIDIA driver package|kernel interface matches" + ``` + + Expected output: + + ``` + kernel interface matches. + Found NVIDIA driver package nvidia-modules-6.12.81 + ``` + + If you instead see `Installing the Flatcar kernel sources...` or + `Compiling NVIDIA driver kernel modules...`, the package did not match the running + kernel and the driver is being rebuilt from source (check that the node's Flatcar + version matches the image's `FLATCAR_VERSION`). + +1. Confirm the driver is loaded: + + ```bash + kubectl -n gpu-operator exec ds/nvidia-driver-daemonset -- nvidia-smi + ``` + +1. On NVSwitch based systems (e.g. HGX H100 8-GPU), confirm the Fabric Manager started and the + fabric is initialized: + + ```bash + kubectl -n gpu-operator logs -l app=nvidia-driver-daemonset | grep "fabric manager" + # Starting NVIDIA fabric manager daemon... + + kubectl -n gpu-operator exec ds/nvidia-driver-daemonset -- nvidia-smi -q | grep -A2 "^ Fabric" + # Fabric + # State : Completed + # Status : Success + ``` + +1. Confirm the whole stack converges — all pods `Running` and the CUDA validators + `Completed`: + + ```bash + kubectl -n gpu-operator get pods + ``` + +### Reinstalling from scratch + +If you delete the `gpu-operator` namespace without `helm uninstall`, cluster-scoped +leftovers block a fresh install. Clean them up first: + +```bash +kubectl delete clusterpolicy cluster-policy +kubectl delete clusterrole,clusterrolebinding gpu-operator \ + gpu-operator-node-feature-discovery gpu-operator-node-feature-discovery-gc +kubectl delete crd nodefeatures.nfd.k8s-sigs.io nodefeaturerules.nfd.k8s-sigs.io \ + nodefeaturegroups.nfd.k8s-sigs.io clusterpolicies.nvidia.com nvidiadrivers.nvidia.com +``` + +### Switching to a new driver image + +To roll out a newly built image without reinstalling: + +```bash +kubectl patch clusterpolicy cluster-policy --type merge \ + -p '{"spec":{"driver":{"version":"flatcar-test--595.71.05"}}}' +kubectl -n gpu-operator delete pods -l app=nvidia-driver-daemonset +``` + +## Sample CUDA Workloads + +Now we can run some sample CUDA workloads. + +1. Run `nvidia-smi` + ```bash + $ docker run --runtime=nvidia nvidia/cuda:11.0-base nvidia-smi + + +-----------------------------------------------------------------------------+ + | NVIDIA-SMI 460.32.03 Driver Version: 460.32.03 CUDA Version: 11.2 | + |-------------------------------+----------------------+----------------------+ + | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | + | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | + | | | MIG M. | + |===============================+======================+======================| + | 0 Tesla T4 On | 00000000:00:1E.0 Off | 0 | + | N/A 30C P8 9W / 70W | 0MiB / 15109MiB | 0% Default | + | | | N/A | + +-------------------------------+----------------------+----------------------+ + + +-----------------------------------------------------------------------------+ + | Processes: | + | GPU GI CI PID Type Process name GPU Memory | + | ID ID Usage | + |=============================================================================| + | No running processes found | + +-----------------------------------------------------------------------------+ + ``` + +1. Run a sample CUDA program: + ```bash + $ docker run --runtime=nvidia nvidia/samples:vectoradd-cuda11.2.1 + + [Vector addition of 50000 elements] + Copy input data from the host memory to the CUDA device + CUDA kernel launch with 196 blocks of 256 threads + Copy output data from the CUDA device to the host memory + Test PASSED + Done + ``` + +1. Generate a deterministic FP16 GEMM on the GPU using the Tensor Cores if available: + ```bash + $ docker run --runtime=nvidia \ + --cap-add SYS_ADMIN \ + nvidia/samples:dcgmproftester-2.0.10-cuda11.0-ubuntu18.04 \ + --no-dcgm-validation -t 1004 -d 30 + + Skipping CreateDcgmGroups() since DCGM validation is disabled + CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR: 1024 + CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT: 40 + CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR: 65536 + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR: 7 + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR: 5 + CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH: 256 + CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE: 5001000 + Max Memory bandwidth: 320064000000 bytes (320.06 GiB) + CudaInit completed successfully. + + Skipping WatchFields() since DCGM validation is disabled + TensorEngineActive: generated ???, dcgm 0.000 (26723.0 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27450.3 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27520.2 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27386.5 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27294.1 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27304.2 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27220.1 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27193.2 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27139.5 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27125.2 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (26985.2 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27282.3 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27413.9 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27366.0 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27378.1 gflops) + TensorEngineActive: generated ???, dcgm 0.000 (27244.6 gflops) + ``` diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..393725c --- /dev/null +++ b/build.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -xeuo pipefail + +: ${DRIVER_VERSION:="$(sed -n "s/^ARG DRIVER_VERSION=//p" Dockerfile)"} + +docker build --pull \ + --tag nvidia/nvidia-driver-flatcar:${DRIVER_VERSION} \ + --file Dockerfile . +docker tag nvidia/nvidia-driver-flatcar:${DRIVER_VERSION} nvidia/nvidia-driver-flatcar:latest diff --git a/empty b/empty new file mode 100644 index 0000000..e69de29 diff --git a/nvidia-driver b/nvidia-driver new file mode 100755 index 0000000..903954c --- /dev/null +++ b/nvidia-driver @@ -0,0 +1,552 @@ +#! /bin/bash +# Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. + +set -eu + +RUN_DIR=/run/nvidia +PID_FILE=${RUN_DIR}/${0##*/}.pid +DRIVER_VERSION=${DRIVER_VERSION:?"Missing driver version"} +#NVIDIA_BINUITLS_DIR=/opt/nvidia/binutils +#NVIDIA_KMODS_DIR=/opt/nvidia/${DRIVER_VERSION} +GPU_DIRECT_RDMA_ENABLED=${GPU_DIRECT_RDMA_ENABLED:-false} +CREATE_PRECOMPILED_PACKAGE=${CREATE_PRECOMPILED_PACKAGE:-false} + + +OPEN_KERNEL_MODULES_ENABLED=${OPEN_KERNEL_MODULES_ENABLED:-false} +[[ "${OPEN_KERNEL_MODULES_ENABLED}" == "true" ]] && KERNEL_TYPE=kernel-open || KERNEL_TYPE=kernel + +#export COREOS_RELEASE_CHANNEL=stable +#COREOS_RELEASE_BOARD=amd64-usr +#COREOS_ALL_RELEASES="https://www.flatcar-linux.org/releases-json/releases-${COREOS_RELEASE_CHANNEL}.json" + +export DEBIAN_FRONTEND=noninteractive + +_relocate_flatcar() { + set -euo pipefail + + #shopt -s nullglob + + hostlds=( /host-lib/usr/lib64/ld-linux-*.so.* ) + if [[ ${#hostlds[@]} -eq 0 ]]; then + echo "** no dynamic loaders found" + return 1 + fi + if [[ ${#hostlds[@]} -gt 1 ]]; then + echo "** more than one fitting dynamic loader found, picking first" + fi + hostld=${hostlds[0]} + echo "** Found host dynamic loader: ${hostld}" + + kdirs=( /lib/modules/${KERNEL_VERSION}/build ) + if [[ ${#kdirs[@]} -eq 0 ]]; then + echo "** no kernel module tools directories found" + return 1 + fi + if [[ ${#kdirs[@]} -gt 1 ]]; then + echo "** more than one fitting kernel module tools directory found, picking first" + fi + kdir=${kdirs[0]} + echo "** Found kernel tools directory: ${kdir}" + + tools=( + scripts/basic/fixdep + scripts/mod/modpost + tools/objtool/objtool + ) + + for tool in "${tools[@]}"; do + host_tool=${kdir}/${tool} + if [[ ! -f ${host_tool} ]]; then + echo "${tool@Q} not found in ${kdir@Q}, not patching" + continue + fi + echo "** Setting host dynamic loader for ${tool@Q}" + patchelf \ + --set-interpreter "${hostld}" \ + --set-rpath /host-lib/usr/lib64 \ + "${host_tool}" + done +} + +_is_flatcar() { + nsenter -t 1 -m grep -q 'ID=flatcar' /etc/os-release +} + +# Check if the system is NVSwitch based (e.g. HGX/DGX A100, H100, H200). +_assert_nvswitch_system() { + [ -d /proc/driver/nvidia-nvswitch/devices ] || return 1 + if [ -z "$(ls -A /proc/driver/nvidia-nvswitch/devices)" ]; then + return 1 + fi + return 0 +} + + +_fetch_flatcar_modules() { + local versionid=${1-} + if [ -z "$versionid" ]; then + versionid=$(. /host-etc/os-release; echo $VERSION_ID) + fi + local channel= + local board= + case "$versionid" in + *.2.*) + channel=stable;; + *.1.*) + channel=beta;; + *.0.*) + channel=alpha;; + esac + + case "$(uname -m)" in + x86_64) + board=amd64-usr;; + aarch64) + board=arm64-usr;; + esac + echo "Installing the Flatcar kernel sources into the development environment..." + + mkdir -p /host-lib + mkdir -p /lib/modules + + curl --retry 3 -fLJO "https://${channel}.release.flatcar-linux.net/${board}/${versionid}/flatcar-container.tar.gz" + tar xf flatcar-container.tar.gz -C /lib/modules ./usr/lib/modules --strip-components 4 + tar xf flatcar-container.tar.gz -C /host-lib ./usr/lib64 --exclude=./usr/lib64/misc/* + KERNEL_VERSION="$(ls /lib/modules/)" + + export FLATCAR_RELEASE_CHANNEL="${channel}" + export FLATCAR_RELEASE_BOARD="${board}" + export FLATCAR_RELEASE_VERSION_ID="${versionid}" +} + +_extract_flatcar_proc_version() { + local versionid=${1:?Missing Flatcar version} + local channel=${2:?Missing Flatcar channel} + local board=${3:?Missing Flatcar board} + local image_path=/tmp/flatcar_production_image.vmlinuz + local banners_path=/tmp/flatcar_linux_banners + local proc_version_path=/lib/modules/${KERNEL_VERSION}/proc/version + + curl --retry 3 -fsSL -o "${image_path}" \ + "https://${channel}.release.flatcar-linux.net/${board}/${versionid}/flatcar_production_image.vmlinuz" + + # Buffer all banner candidates into a file first: grep -m1 on the live + # pipeline would close the pipe early and kill extract-vmlinux/strings + # with SIGPIPE (pipefail is enabled by _relocate_flatcar). + extract-vmlinux "${image_path}" | strings | grep -E '^Linux version ' > "${banners_path}" || true + + # The image embeds multiple "Linux version" strings. Only the real banner + # (identical to /proc/version on the booted host) carries a build number + # and timestamp, e.g. "... #1 SMP PREEMPT_DYNAMIC Mon Jun 15 08:27:43 -00 2026". + # A truncated template variant ("... # SMP PREEMPT_DYNAMIC ") appears first + # in the image and must be skipped, otherwise the packed proc version never + # matches at runtime and the driver is rebuilt from source. + if ! grep -m1 -E "^Linux version ${KERNEL_VERSION} .*#[0-9]+ SMP " "${banners_path}" > "${proc_version_path}"; then + grep -m1 -E '^Linux version .*#[0-9]+ SMP ' "${banners_path}" > "${proc_version_path}" || true + fi + + if [ ! -s "${proc_version_path}" ]; then + echo "Could not derive Flatcar kernel version string from flatcar_production_image.vmlinuz" >&2 + return 1 + fi + + echo "Using Flatcar proc/version: $(cat "${proc_version_path}")" +} + +_fetch_flatcar_modules_host() { + mkdir -p /host-lib + mkdir -p /lib/modules + nsenter -t 1 -m tar cpf - /lib/modules/${KERNEL_VERSION} | tar xpf - -C / + nsenter -t 1 -m tar cpf - /usr/lib64/ | tar xpf - -C /host-lib/ +} + +_install_prerequisites() { + if [ -n "${FLATCAR_VERSION}" ]; then + _fetch_flatcar_modules "${FLATCAR_VERSION}" + elif _is_flatcar; then + _fetch_flatcar_modules_host + else + _fetch_flatcar_modules + fi + + rm -rf /usr/src/linux* + #rm -rf /host-lib + + # Gather the tag for the release matching the current kernel version. + #local kernel + #kernel=$(echo "${KERNEL_VERSION}" | cut -d "-" -f1) + #export COREOS_RELEASE_VERSION=$(curl -Ls "${COREOS_ALL_RELEASES}" | jq -r --arg kernel_ver "${kernel}" 'to_entries[] | select ((.value.major_software.kernel[0] == $kernel_ver) and (.key != "current")) | .key') + + _relocate_flatcar + + mkdir -p /lib/modules/${KERNEL_VERSION}/proc + if [ -n "${FLATCAR_VERSION}" ]; then + _extract_flatcar_proc_version "${FLATCAR_RELEASE_VERSION_ID}" "${FLATCAR_RELEASE_CHANNEL}" "${FLATCAR_RELEASE_BOARD}" + else + cp /proc/version /lib/modules/${KERNEL_VERSION}/proc + fi +} + +# Check if the kernel version requires a new precompiled driver packages. +_kernel_requires_package() { + local proc_mount_arg="" + + echo "Checking NVIDIA driver packages..." + cd "/usr/src/nvidia-${DRIVER_VERSION}/${KERNEL_TYPE}" + + # Prefer the proc/version baked into the precompiled image (written at pack time) + # over the running host's /proc/version. When both are the same build the files + # are identical; when KERNEL_VERSION == uname -r (normal Flatcar runtime) the old + # code left proc_mount_arg empty and mkprecompiled fell back to the host's + # /proc/version, which may differ from the vmlinuz-extracted string we packed with. + if [ -d "/lib/modules/${KERNEL_VERSION}/proc" ] || [ "${KERNEL_VERSION}" != "$(uname -r)" ]; then + proc_mount_arg="--proc-mount-point /lib/modules/${KERNEL_VERSION}/proc" + fi + if [ -n "${proc_mount_arg}" ]; then + echo "Using proc/version for match: $(cat /lib/modules/${KERNEL_VERSION}/proc/version 2>/dev/null || echo '')" + else + echo "Using host /proc/version for match: $(cat /proc/version)" + fi + # Note that mkprecompiled does not handle empty double-quotes; + # so don't wrap variables in double-quotes + # The precompiled directory structure is laid out as one directory per KERNEL_VERSION + # and it includes the packed kernel interfaces and other files. So we use the following search pattern: + for pkg_name in $(ls -d -1 precompiled/** 2> /dev/null); do + if ! ../mkprecompiled --match ${pkg_name} ${proc_mount_arg} ; then + echo "Found NVIDIA driver package ${pkg_name##*/}" + return 1 + fi + done + return 0 +} + +# Compile the kernel modules, optionally sign them, and generate a precompiled package for use later. +_create_driver_package() { + if [ "${CREATE_PRECOMPILED_PACKAGE}" != "true" ]; then + return 0 + fi + + local pkg_name="nvidia-modules-${KERNEL_VERSION%%-*}${PACKAGE_TAG:+-${PACKAGE_TAG}}" + local nvidia_sign_args="" + local nvidia_modeset_sign_args="" + local nvidia_uvm_sign_args="" + local nvidia_peermem_sign_args="" + + trap "make -s -j SYSSRC=/lib/modules/${KERNEL_VERSION}/build clean > /dev/null" EXIT + echo "Compiling NVIDIA driver kernel modules..." + cd "/usr/src/nvidia-${DRIVER_VERSION}/${KERNEL_TYPE}" + + if [ -n "${PRIVATE_KEY}" ]; then + echo "Signing NVIDIA driver kernel modules..." + donkey get "${PRIVATE_KEY}" sh -c "PATH=${PATH}:/usr/src/linux-headers-${KERNEL_VERSION}/scripts && \ + sign-file sha512 \$DONKEY_FILE pubkey.x509 nvidia.ko nvidia.ko.sign && \ + sign-file sha512 \$DONKEY_FILE pubkey.x509 nvidia-modeset.ko nvidia-modeset.ko.sign && \ + sign-file sha512 \$DONKEY_FILE pubkey.x509 nvidia-uvm.ko && + sign-file sha512 \$DONKEY_FILE pubkey.x509 nvidia-peermem.ko" + nvidia_sign_args="--linked-module nvidia.ko --signed-module nvidia.ko.sign" + nvidia_modeset_sign_args="--linked-module nvidia-modeset.ko --signed-module nvidia-modeset.ko.sign" + nvidia_uvm_sign_args="--signed" + nvidia_peermem_sign_args="--signed" + fi + + export IGNORE_CC_MISMATCH=1 + make -s -j$(nproc) SYSSRC=/lib/modules/${KERNEL_VERSION}/build nv-linux.o nv-modeset-linux.o > /dev/null + + # Since kernel 6.12 the vermagic (and other module boilerplate) lives in a + # separate module-common object that kbuild only links into the final .ko + # during the modfinal stage. Merge it into the kernel interfaces before + # packing, otherwise modules relinked from the precompiled interfaces lack + # vermagic and the kernel rejects them with "Invalid module format". + if [ -f .module-common.o ]; then + echo "Merging .module-common.o into the kernel interfaces..." + ld -r -o nv-linux-common.o nv-linux.o .module-common.o && mv nv-linux-common.o nv-linux.o + ld -r -o nv-modeset-linux-common.o nv-modeset-linux.o .module-common.o && mv nv-modeset-linux-common.o nv-modeset-linux.o + fi + + find -name '*.ko' + echo "Relinking NVIDIA driver kernel modules..." + rm -f nvidia.ko nvidia-modeset.ko + ld -d -r -o nvidia.ko ./nv-linux.o ./nvidia/nv-kernel.o_binary + ld -d -r -o nvidia-modeset.ko ./nv-modeset-linux.o ./nvidia-modeset/nv-modeset-kernel.o_binary + + # Note that mkprecompiled does not handle empty double-quotes; + # so don't wrap variables in double-quotes + echo "Building NVIDIA driver package ${pkg_name}..." + ../mkprecompiled --pack ${pkg_name} --description ${KERNEL_VERSION} \ + --proc-mount-point /lib/modules/${KERNEL_VERSION}/proc \ + --driver-version ${DRIVER_VERSION} \ + --kernel-interface nv-linux.o \ + --linked-module-name nvidia.ko \ + --core-object-name nvidia/nv-kernel.o_binary \ + ${nvidia_sign_args} \ + --target-directory . \ + --kernel-interface nv-modeset-linux.o \ + --linked-module-name nvidia-modeset.ko \ + --core-object-name nvidia-modeset/nv-modeset-kernel.o_binary \ + ${nvidia_modeset_sign_args} \ + --target-directory . \ + --kernel-module nvidia-uvm.ko \ + ${nvidia_uvm_sign_args} \ + --target-directory . \ + --kernel-module nvidia-peermem.ko \ + ${nvidia_peermem_sign_args} \ + --target-directory . + mkdir -p precompiled + mv ${pkg_name} precompiled +} + +# Load the kernel modules and start persistenced. +_load_driver() { + echo "Loading NVIDIA driver kernel modules..." + modprobe -a nvidia nvidia-uvm nvidia-modeset + + echo "Starting NVIDIA persistence daemon..." + nvidia-persistenced --persistence-mode + + if _assert_nvswitch_system; then + echo "Starting NVIDIA fabric manager daemon..." + nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg + fi +} + +# Stop persistenced and unload the kernel modules if they are currently loaded. +_unload_driver() { + local rmmod_args=() + local nvidia_deps=0 + local nvidia_refs=0 + local nvidia_uvm_refs=0 + local nvidia_modeset_refs=0 + + echo "Stopping NVIDIA persistence daemon..." + if [ -f /var/run/nvidia-persistenced/nvidia-persistenced.pid ]; then + local pid + pid=$(< /var/run/nvidia-persistenced/nvidia-persistenced.pid) + + kill -SIGTERM "${pid}" + for i in $(seq 1 10); do + kill -0 "${pid}" 2> /dev/null || break + sleep 0.1 + done + if [ "$i" -eq 10 ]; then + echo "Could not stop NVIDIA persistence daemon" >&2 + return 1 + fi + fi + + if [ -f /var/run/nvidia-fabricmanager/nv-fabricmanager.pid ]; then + echo "Stopping NVIDIA fabric manager daemon..." + local pid + pid=$(< /var/run/nvidia-fabricmanager/nv-fabricmanager.pid) + + kill -SIGTERM "${pid}" + for i in $(seq 1 50); do + kill -0 "${pid}" 2> /dev/null || break + sleep 0.1 + done + if [ "$i" -eq 50 ]; then + echo "Could not stop NVIDIA fabric manager daemon" >&2 + return 1 + fi + fi + + echo "Unloading NVIDIA driver kernel modules..." + if [ -f /sys/module/nvidia_modeset/refcnt ]; then + nvidia_modeset_refs=$(< /sys/module/nvidia_modeset/refcnt) + rmmod_args+=("nvidia-modeset") + ((++nvidia_deps)) + fi + if [ -f /sys/module/nvidia_uvm/refcnt ]; then + nvidia_uvm_refs=$(< /sys/module/nvidia_uvm/refcnt) + rmmod_args+=("nvidia-uvm") + ((++nvidia_deps)) + fi + if [ -f /sys/module/nvidia/refcnt ]; then + nvidia_refs=$(< /sys/module/nvidia/refcnt) + rmmod_args+=("nvidia") + fi + if [ "${nvidia_refs}" -gt "${nvidia_deps}" ] || [ "${nvidia_uvm_refs}" -gt 0 ] || \ + [ "${nvidia_modeset_refs}" -gt 0 ]; then + echo "Could not unload NVIDIA driver kernel modules, driver is in use" >&2 + return 1 + fi + + if [ ${#rmmod_args[@]} -gt 0 ]; then + rmmod "${rmmod_args[@]}" + fi + return 0 +} + +_install_driver() { + local install_args=() + + echo "Installing NVIDIA driver kernel modules..." + cd /usr/src/nvidia-${DRIVER_VERSION} + rm -rf /lib/modules/${KERNEL_VERSION}/video + + if [ "${ACCEPT_LICENSE}" = "yes" ]; then + install_args+=("--accept-license") + fi + if [ "${KERNEL_VERSION}" != "$(uname -r)" ]; then + # Passing this arg prevents use of a precompiled package. + install_args+=(--kernel-name "${KERNEL_VERSION}") + fi + IGNORE_CC_MISMATCH=1 nvidia-installer --kernel-module-only --no-drm --ui=none --no-nouveau-check -m=${KERNEL_TYPE} ${install_args[@]+"${install_args[@]}"} +} + +# Execute binaries by root owning them first +_exec() { + exec_bin_path=$(command -v "$1") + exec_user=$(stat -c "%u" "${exec_bin_path}") + exec_group=$(stat -c "%g" "${exec_bin_path}") + if [[ "${exec_user}" != "0" || "${exec_group}" != "0" ]]; then + chown 0:0 "${exec_bin_path}" + "$@" + chown "${exec_user}":"${exec_group}" "${exec_bin_path}" + else + "$@" + fi +} + +# Mount the driver rootfs into the run directory with the exception of sysfs. +_mount_rootfs() { + echo "Mounting NVIDIA driver rootfs..." + _exec mount --make-runbindable /sys + _exec mount --make-private /sys + mkdir -p ${RUN_DIR}/driver + _exec mount --rbind / ${RUN_DIR}/driver + + echo "Check SELinux status" + if [ -e /sys/fs/selinux ]; then + echo "SELinux is enabled" + echo "Change device files security context for selinux compatibility" + chcon -R -t container_file_t ${RUN_DIR}/driver/dev + else + echo "SELinux is disabled, skipping..." + fi +} + +# Unmount the driver rootfs from the run directory. +_unmount_rootfs() { + echo "Unmounting NVIDIA driver rootfs..." + if findmnt -r -o TARGET | grep "${RUN_DIR}/driver" > /dev/null; then + _exec umount -l -R ${RUN_DIR}/driver + fi +} + +_shutdown() { + if _unload_driver; then + _unmount_rootfs + rm -f "${PID_FILE}" + return 0 + fi + return 1 +} + +init() { + printf "\\n========== NVIDIA Software Installer ==========\\n" + printf "Starting installation of NVIDIA driver version %s for Linux kernel version %s\\n" "${DRIVER_VERSION}" "${KERNEL_VERSION}" + + exec 3> "${PID_FILE}" + if ! flock -n 3; then + echo "An instance of the NVIDIA driver is already running, aborting" + exit 1 + fi + echo $$ >&3 + + trap "echo 'Caught signal'; exit 1" HUP INT QUIT PIPE TERM + trap "_shutdown" EXIT + + _unload_driver || exit 1 + _unmount_rootfs + + if _kernel_requires_package; then + _install_prerequisites + _create_driver_package + fi + + _install_driver + _load_driver + _mount_rootfs + + echo "Done, now waiting for signal" + sleep infinity & + trap "echo 'Caught signal'; _shutdown && { kill $!; exit 0; }" HUP INT QUIT PIPE TERM + trap - EXIT + while true; do wait $! || continue; done + exit 0 +} + +update() { + exec 3>&2 + if exec 2> /dev/null 4< "${PID_FILE}"; then + if ! flock -n 4 && read -r pid <&4 && kill -0 "${pid}"; then + exec > >(tee -a "/proc/${pid}/fd/1") + exec 2> >(tee -a "/proc/${pid}/fd/2" >&3) + else + exec 2>&3 + fi + exec 4>&- + fi + exec 3>&- + + printf "\\n========== NVIDIA Software Updater ==========\\n" + printf "Starting update of NVIDIA driver version %s for Linux kernel version %s\\n" "${DRIVER_VERSION}" "${KERNEL_VERSION}" + + trap "echo 'Caught signal'; exit 1" HUP INT QUIT PIPE TERM + + _install_prerequisites + + if _kernel_requires_package; then + _create_driver_package + fi + + echo "Done" + exit 0 +} + +usage() { + cat >&2 <