Security hardening: pinned versions, locked deps, CUDA/PyTorch upgrades

.github/workflows/dev.yml

@@ -36,13 +36,13 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build and push 5090 dev image
+      - name: Build and push CUDA 13.0 dev image
         uses: docker/bake-action@v5
         with:
           push: ${{ inputs.push }}
           files: ./docker-bake.hcl
           targets: |
-            devpush5090
+            devpush-cuda13
 
       - name: Clean up Docker build cache
         run: |

.github/workflows/release.yml

@@ -67,15 +67,15 @@ jobs:
           # Export TAG for docker-bake.hcl variable override
           echo "TAG=${RELEASE_VERSION}" >> $GITHUB_ENV
 
-      - name: Build and push 5090 image
+      - name: Build and push CUDA 13.0 image
         uses: docker/bake-action@v5
         env:
           TAG: ${{ env.RELEASE_VERSION }}
         with:
           push: true
           files: ./docker-bake.hcl
           targets: |
-            rtx5090
+            cuda13
 
       - name: Clean up Docker build cache
         run: |
@@ -107,10 +107,11 @@ jobs:
           echo "🚀 Release completed!"
           echo "Version: ${{ env.RELEASE_VERSION }}"
           echo "Docker Images:"
-          echo " - ${{ env.IMAGE_REF }}:${{ env.RELEASE_VERSION }} (regular)"
-          echo " - ${{ env.IMAGE_REF }}:latest (regular)"
-          echo " - ${{ env.IMAGE_REF }}:${{ env.RELEASE_VERSION }}-5090 (RTX 5090)"
-          echo " - ${{ env.IMAGE_REF }}:latest-5090 (RTX 5090)"
+          echo " - ${{ env.IMAGE_REF }}:${{ env.RELEASE_VERSION }}-cuda12.8 (CUDA 12.8)"
+          echo " - ${{ env.IMAGE_REF }}:cuda12.8"
+          echo " - ${{ env.IMAGE_REF }}:latest"
+          echo " - ${{ env.IMAGE_REF }}:${{ env.RELEASE_VERSION }}-cuda13.0 (CUDA 13.0)"
+          echo " - ${{ env.IMAGE_REF }}:cuda13.0"
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             echo "Trigger: Manual workflow dispatch"
           else

CHANGELOG.md

@@ -0,0 +1,54 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [Unreleased]
+
+### Breaking Changes
+
+- **Venv path renamed** from `.venv` to `.venv-cu128`. Existing users on the legacy (non-Blackwell) template will see a one-time re-setup on first boot after upgrading. Blackwell and newer users are unaffected.
+- **Ubuntu 22.04 → 24.04** for both images. Python 3.12 is now provided by the base OS (deadsnakes PPA no longer needed).
+- **CUDA upgraded**: regular image now uses CUDA 12.8 (cu128 wheels), CUDA 13.0 image uses cu130 wheels. CUDA 12.4 is no longer supported.
+- **No runtime dependency installs**: all Python dependencies are baked into the image at build time. The start script no longer runs `pip install` or executes `install.py`/`setup.py` on boot. Custom nodes installed at runtime via ComfyUI-Manager are the user's responsibility; their dependencies persist in the venv across reboots.
+- **Dockerfile.5090 and start.5090.sh removed**: a single `Dockerfile` and `start.sh` now serve all variants, controlled by build args in `docker-bake.hcl`.
+- **ComfyUI runs in foreground**: `exec python main.py` replaces the old `nohup` + `tail -f` pattern. Logs go directly to container stdout instead of `/workspace/runpod-slim/comfyui.log`.
+- **Docker image tag scheme changed**: tags now use CUDA version instead of GPU model name. See new tag scheme below.
+
+### New Docker Image Tag Scheme
+
+Tags now clearly identify the CUDA version. The old `5090`-suffixed tags are deprecated.
+
+On each release (e.g. `2.0.0`):
+
+| Tag | Description |
+|---|---|
+| `runpod/comfyui:2.0.0-cuda12.8` | Pinned release, CUDA 12.8 |
+| `runpod/comfyui:2.0.0-cuda13.0` | Pinned release, CUDA 13.0 |
+| `runpod/comfyui:cuda12.8` | Always latest CUDA 12.8 build |
+| `runpod/comfyui:cuda13.0` | Always latest CUDA 13.0 build |
+| `runpod/comfyui:latest` | Always latest CUDA 12.8 (default) |
+
+**Deprecated tags** (no longer produced):
+- `runpod/comfyui:*-5090`
+- `runpod/comfyui:latest-5090`
+
+### Added
+
+- Centralized version pinning in `docker-bake.hcl` (single source of truth for ComfyUI, custom node SHAs, PyTorch, FileBrowser).
+- Hash-verified dependency lock file generated at build time via `pip-compile --generate-hashes`.
+- `scripts/fetch-hashes.sh` to query GitHub API for latest custom node commit SHAs.
+- `scripts/prebake-manager-cache.py` to pre-populate ComfyUI-Manager cache at build time, reducing cold start time.
+- ComfyUI-RunpodDirect added as a pre-installed custom node.
+- Git init with tagged commits and upstream remotes at build time so ComfyUI-Manager can detect versions.
+- FileBrowser pinned to a specific version with SHA256 checksum verification.
+- PyTorch 2.10.0 + torchvision 0.25.0 + torchaudio 2.10.0 for both images.
+- Separate PyTorch version pins for regular and CUDA 13.0 images so versions can diverge independently.
+
+### Removed
+
+- `Dockerfile.5090` (unified into `Dockerfile` with build args)
+- `start.5090.sh` (unified into `start.sh`)
+- Runtime `git clone` and `pip install` loops from start script
+- `golang` and `make` from runtime image (no longer needed without FileBrowser build-from-source)
+- deadsnakes PPA and ffmpeg-nvenc PPA dependencies
+- `5090`-suffixed Docker image tags (replaced by CUDA-versioned tags)

Dockerfile

@@ -1,91 +1,138 @@
 # ============================================================================
-# Stage 1: Builder - Clone ComfyUI and install all Python packages
+# Stage 1: Builder - Download pinned sources and install all Python packages
 # ============================================================================
-FROM ubuntu:22.04 AS builder
+FROM ubuntu:24.04 AS builder
 
 ENV DEBIAN_FRONTEND=noninteractive
 
+# ---- Version pins (set in docker-bake.hcl) ----
+ARG COMFYUI_VERSION
+ARG MANAGER_SHA
+ARG KJNODES_SHA
+ARG CIVICOMFY_SHA
+ARG RUNPODDIRECT_SHA
+ARG TORCH_VERSION
+ARG TORCHVISION_VERSION
+ARG TORCHAUDIO_VERSION
+
+# ---- CUDA variant (set in docker-bake.hcl per target) ----
+ARG CUDA_VERSION_DASH=12-8
+ARG TORCH_INDEX_SUFFIX=cu128
+
 # Install minimal dependencies needed for building
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    software-properties-common \
-    gpg-agent \
-    git \
     wget \
     curl \
+    git \
     ca-certificates \
-    && add-apt-repository ppa:deadsnakes/ppa && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
     python3.12 \
     python3.12-venv \
     python3.12-dev \
     build-essential \
-    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \
     && dpkg -i cuda-keyring_1.1-1_all.deb \
     && apt-get update \
-    && apt-get install -y --no-install-recommends cuda-minimal-build-12-4 \
+    && apt-get install -y --no-install-recommends cuda-minimal-build-${CUDA_VERSION_DASH} libcusparse-dev-${CUDA_VERSION_DASH} \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
-    && rm cuda-keyring_1.1-1_all.deb
+    && rm cuda-keyring_1.1-1_all.deb \
+    && rm -f /usr/lib/python3.12/EXTERNALLY-MANAGED
 
-# Install pip for Python 3.12 and upgrade it
+# Install pip and pip-tools for lock file generation
 RUN curl -sS https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
     python3.12 get-pip.py && \
-    python3.12 -m pip install --upgrade pip && \
+    python3.12 -m pip install --no-cache-dir pip-tools && \
     rm get-pip.py
 
 # Set CUDA environment for building
 ENV PATH=/usr/local/cuda/bin:${PATH}
 ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64
 
-# Clone ComfyUI to get requirements
+# Download pinned source archives
 WORKDIR /tmp/build
-RUN git clone https://github.com/comfyanonymous/ComfyUI.git
+RUN curl -fSL "https://github.com/comfyanonymous/ComfyUI/archive/refs/tags/${COMFYUI_VERSION}.tar.gz" -o comfyui.tar.gz && \
+    mkdir -p ComfyUI && tar xzf comfyui.tar.gz --strip-components=1 -C ComfyUI && rm comfyui.tar.gz
 
-# Clone custom nodes to get their requirements
 WORKDIR /tmp/build/ComfyUI/custom_nodes
-RUN git clone https://github.com/ltdrdata/ComfyUI-Manager.git && \
-    git clone https://github.com/kijai/ComfyUI-KJNodes && \
-    git clone https://github.com/MoonGoblinDev/Civicomfy
-
-# Install PyTorch and all ComfyUI dependencies
+RUN curl -fSL "https://github.com/ltdrdata/ComfyUI-Manager/archive/${MANAGER_SHA}.tar.gz" -o manager.tar.gz && \
+    mkdir -p ComfyUI-Manager && tar xzf manager.tar.gz --strip-components=1 -C ComfyUI-Manager && rm manager.tar.gz && \
+    curl -fSL "https://github.com/kijai/ComfyUI-KJNodes/archive/${KJNODES_SHA}.tar.gz" -o kjnodes.tar.gz && \
+    mkdir -p ComfyUI-KJNodes && tar xzf kjnodes.tar.gz --strip-components=1 -C ComfyUI-KJNodes && rm kjnodes.tar.gz && \
+    curl -fSL "https://github.com/MoonGoblinDev/Civicomfy/archive/${CIVICOMFY_SHA}.tar.gz" -o civicomfy.tar.gz && \
+    mkdir -p Civicomfy && tar xzf civicomfy.tar.gz --strip-components=1 -C Civicomfy && rm civicomfy.tar.gz && \
+    curl -fSL "https://github.com/MadiatorLabs/ComfyUI-RunpodDirect/archive/${RUNPODDIRECT_SHA}.tar.gz" -o runpoddirect.tar.gz && \
+    mkdir -p ComfyUI-RunpodDirect && tar xzf runpoddirect.tar.gz --strip-components=1 -C ComfyUI-RunpodDirect && rm runpoddirect.tar.gz
+
+# Init git repos with upstream remotes so ComfyUI-Manager can detect versions
+# and users can update via Manager at their own risk
+RUN cd /tmp/build/ComfyUI && \
+    git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI ${COMFYUI_VERSION}" && git tag "${COMFYUI_VERSION}" && \
+    git remote add origin https://github.com/comfyanonymous/ComfyUI.git && \
+    cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-Manager && \
+    git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-Manager ${MANAGER_SHA}" && \
+    git remote add origin https://github.com/ltdrdata/ComfyUI-Manager.git && \
+    cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-KJNodes && \
+    git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-KJNodes ${KJNODES_SHA}" && \
+    git remote add origin https://github.com/kijai/ComfyUI-KJNodes.git && \
+    cd /tmp/build/ComfyUI/custom_nodes/Civicomfy && \
+    git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "Civicomfy ${CIVICOMFY_SHA}" && \
+    git remote add origin https://github.com/MoonGoblinDev/Civicomfy.git && \
+    cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-RunpodDirect && \
+    git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-RunpodDirect ${RUNPODDIRECT_SHA}" && \
+    git remote add origin https://github.com/MadiatorLabs/ComfyUI-RunpodDirect.git
+
+# Install PyTorch (pinned version)
 RUN python3.12 -m pip install --no-cache-dir \
-    torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
-
-WORKDIR /tmp/build/ComfyUI
-RUN python3.12 -m pip install --no-cache-dir -r requirements.txt && \
-    python3.12 -m pip install --no-cache-dir GitPython opencv-python
+    torch==${TORCH_VERSION} torchvision==${TORCHVISION_VERSION} torchaudio==${TORCHAUDIO_VERSION} \
+    --index-url https://download.pytorch.org/whl/${TORCH_INDEX_SUFFIX}
 
-# Install custom node dependencies
-WORKDIR /tmp/build/ComfyUI/custom_nodes
-RUN for node_dir in */; do \
+# Generate lock file from all requirements, then install with hash verification
+WORKDIR /tmp/build
+RUN cat ComfyUI/requirements.txt > requirements.in && \
+    for node_dir in ComfyUI/custom_nodes/*/; do \
         if [ -f "$node_dir/requirements.txt" ]; then \
-            echo "Installing requirements for $node_dir"; \
-            python3.12 -m pip install --no-cache-dir -r "$node_dir/requirements.txt" || true; \
+            cat "$node_dir/requirements.txt" >> requirements.in; \
         fi; \
-    done
+    done && \
+    echo "GitPython" >> requirements.in && \
+    echo "opencv-python" >> requirements.in && \
+    echo "jupyter" >> requirements.in && \
+    echo "torch==${TORCH_VERSION}" >> constraints.txt && \
+    echo "torchvision==${TORCHVISION_VERSION}" >> constraints.txt && \
+    echo "torchaudio==${TORCHAUDIO_VERSION}" >> constraints.txt && \
+    echo "pillow>=12.1.1" >> constraints.txt && \
+    PIP_CONSTRAINT=constraints.txt pip-compile --generate-hashes --output-file=requirements.lock --strip-extras --allow-unsafe requirements.in && \
+    python3.12 -m pip install --no-cache-dir --ignore-installed --require-hashes -r requirements.lock
+
+# Pre-populate ComfyUI-Manager cache so first cold start skips the slow registry fetch
+COPY scripts/prebake-manager-cache.py /tmp/prebake-manager-cache.py
+RUN python3.12 /tmp/prebake-manager-cache.py /tmp/build/ComfyUI/user/__manager/cache
+
+# Bake ComfyUI + custom nodes into a known location for runtime copy
+RUN cp -r /tmp/build/ComfyUI /opt/comfyui-baked
 
 # ============================================================================
 # Stage 2: Runtime - Clean image with pre-installed packages
 # ============================================================================
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED=1
 ENV IMAGEIO_FFMPEG_EXE=/usr/bin/ffmpeg
 ENV FILEBROWSER_CONFIG=/workspace/runpod-slim/.filebrowser.json
 
+# ---- CUDA variant (re-declared for runtime stage) ----
+ARG CUDA_VERSION_DASH=12-8
+
+# ---- FileBrowser version pin (set in docker-bake.hcl) ----
+ARG FILEBROWSER_VERSION
+ARG FILEBROWSER_SHA256
+
 # Update and install runtime dependencies, CUDA, and common tools
 RUN apt-get update && \
     apt-get upgrade -y && \
     apt-get install -y --no-install-recommends \
-    software-properties-common \
-    gpg-agent \
-    && add-apt-repository ppa:deadsnakes/ppa && \
-    add-apt-repository ppa:cybermax-dexter/ffmpeg-nvenc && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
     git \
     python3.12 \
     python3.12-venv \
@@ -106,34 +153,40 @@ RUN apt-get update && \
     net-tools \
     iputils-ping \
     procps \
-    golang \
-    make \
-    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb \
+    openssl \
+    ffmpeg \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \
     && dpkg -i cuda-keyring_1.1-1_all.deb \
     && apt-get update \
-    && apt-get install -y --no-install-recommends cuda-minimal-build-12-4 \
-    && apt-get install -y --no-install-recommends ffmpeg \
+    && apt-get install -y --no-install-recommends cuda-minimal-build-${CUDA_VERSION_DASH} \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
-    && rm cuda-keyring_1.1-1_all.deb
+    && rm cuda-keyring_1.1-1_all.deb \
+    && rm -f /usr/lib/python3.12/EXTERNALLY-MANAGED
 
-# Copy Python packages and pip executables from builder stage
+# Copy Python packages, executables, and Jupyter data from builder stage
 COPY --from=builder /usr/local/lib/python3.12 /usr/local/lib/python3.12
 COPY --from=builder /usr/local/bin /usr/local/bin
+COPY --from=builder /usr/local/share/jupyter /usr/local/share/jupyter
+
+# Copy baked ComfyUI + custom nodes from builder stage
+COPY --from=builder /opt/comfyui-baked /opt/comfyui-baked
 
 # Remove uv to force ComfyUI-Manager to use pip (uv doesn't respect --system-site-packages properly)
 RUN pip uninstall -y uv 2>/dev/null || true && \
     rm -f /usr/local/bin/uv /usr/local/bin/uvx
 
-# Install FileBrowser
-RUN curl -fsSL https://raw.githubusercontent.com/filebrowser/get/master/get.sh | bash
+# Install FileBrowser (pinned version with checksum)
+RUN curl -fSL "https://github.com/filebrowser/filebrowser/releases/download/${FILEBROWSER_VERSION}/linux-amd64-filebrowser.tar.gz" -o /tmp/fb.tar.gz && \
+    echo "${FILEBROWSER_SHA256}  /tmp/fb.tar.gz" | sha256sum -c - && \
+    tar xzf /tmp/fb.tar.gz -C /usr/local/bin filebrowser && \
+    rm /tmp/fb.tar.gz
 
 # Set CUDA environment variables
 ENV PATH=/usr/local/cuda/bin:${PATH}
 ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64
 
-# Install Jupyter with Python kernel
-RUN pip install jupyter
+# Jupyter is included in the lock file and installed in the builder stage
 
 # Configure SSH for root login
 RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \