From d795b6d413aaafd25e437bf9f84138bacbb0a861 Mon Sep 17 00:00:00 2001 From: "blacksmith-sh[bot]" <157653362+blacksmith-sh[bot]@users.noreply.github.com> Date: Thu, 9 Apr 2026 12:56:06 +0000 Subject: [PATCH 01/10] Migrate workflows to Blacksmith --- .github/workflows/auto-request-same-site.yml | 2 +- .github/workflows/build-and-upload-job.yml | 2 +- .github/workflows/claude-code-review.yml | 2 +- .github/workflows/deploy-infra.yml | 2 +- .github/workflows/deploy-job.yml | 2 +- .github/workflows/fc-test.yml | 2 +- .github/workflows/integration_tests.yml | 2 +- .github/workflows/lint.yml | 4 ++-- .github/workflows/out-of-order-migrations.yml | 2 +- .github/workflows/periodic-test.yml | 2 +- .github/workflows/pr-go-dependbot-cleanup.yml | 2 +- .github/workflows/pr-no-generated-changes.yml | 2 +- .github/workflows/pr-tests-arm64.yml | 4 ++-- .github/workflows/pr-tests.yml | 4 ++-- .github/workflows/pull-request.yml | 2 +- .github/workflows/push-main.yml | 2 +- .github/workflows/validate-openapi.yml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/auto-request-same-site.yml b/.github/workflows/auto-request-same-site.yml index 15d9138dbd..dbb16e9815 100644 --- a/.github/workflows/auto-request-same-site.yml +++ b/.github/workflows/auto-request-same-site.yml @@ -21,7 +21,7 @@ permissions: jobs: assign: - runs-on: ubuntu-latest + runs-on: blacksmith-8vcpu-ubuntu-2404 steps: - name: Get GitHub App installation token id: app diff --git a/.github/workflows/build-and-upload-job.yml b/.github/workflows/build-and-upload-job.yml index dc28dbb379..ef743b9b03 100644 --- a/.github/workflows/build-and-upload-job.yml +++ b/.github/workflows/build-and-upload-job.yml @@ -30,7 +30,7 @@ concurrency: jobs: deploy: name: Build and upload job to the ${{ inputs.environment }} environment - runs-on: ci-builder + runs-on: blacksmith-4vcpu-ubuntu-2404 environment: ${{ inputs.environment }} permissions: contents: read diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 85cd5f7d9c..b1e09402db 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -10,7 +10,7 @@ jobs: claude-review: name: Claude Code Review - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 if: | github.event_name == 'pull_request' || (github.event_name == 'issue_comment' && diff --git a/.github/workflows/deploy-infra.yml b/.github/workflows/deploy-infra.yml index 3811aa508d..0eb7275db1 100644 --- a/.github/workflows/deploy-infra.yml +++ b/.github/workflows/deploy-infra.yml @@ -28,7 +28,7 @@ concurrency: jobs: deploy: name: Deploy Infra to the ${{ inputs.environment }} environment - runs-on: ubuntu-22.04 + runs-on: blacksmith-4vcpu-ubuntu-2204 environment: ${{ inputs.environment }} permissions: contents: read diff --git a/.github/workflows/deploy-job.yml b/.github/workflows/deploy-job.yml index b5e1ee2e82..484936e1db 100644 --- a/.github/workflows/deploy-job.yml +++ b/.github/workflows/deploy-job.yml @@ -31,7 +31,7 @@ concurrency: jobs: deploy: name: Deploy job to the ${{ inputs.environment }} environment - runs-on: ubuntu-22.04 + runs-on: blacksmith-4vcpu-ubuntu-2204 environment: ${{ inputs.environment }} permissions: contents: read diff --git a/.github/workflows/fc-test.yml b/.github/workflows/fc-test.yml index 0d6c7f122a..72d5156cb9 100644 --- a/.github/workflows/fc-test.yml +++ b/.github/workflows/fc-test.yml @@ -4,7 +4,7 @@ on: workflow_call jobs: firecracker: - runs-on: ubuntu-24.04 + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout Code diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index af27a2ae71..c1954ab669 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -9,7 +9,7 @@ on: required: true jobs: integration_tests: - runs-on: infra-tests + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 permissions: contents: read diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 24bb277948..2e399d96e4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,7 +7,7 @@ permissions: jobs: detect-modules: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 outputs: modules: ${{ steps.set-modules.outputs.modules }} steps: @@ -17,7 +17,7 @@ jobs: golangci-lint: needs: detect-modules - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: matrix: modules: ${{ fromJSON(needs.detect-modules.outputs.modules) }} diff --git a/.github/workflows/out-of-order-migrations.yml b/.github/workflows/out-of-order-migrations.yml index 5184e67e0a..ffa3270377 100644 --- a/.github/workflows/out-of-order-migrations.yml +++ b/.github/workflows/out-of-order-migrations.yml @@ -5,7 +5,7 @@ on: [workflow_call] jobs: check: name: Check for out of order migrations - runs-on: ubuntu-22.04 + runs-on: blacksmith-4vcpu-ubuntu-2204 steps: - name: Checkout repository uses: actions/checkout@v5 diff --git a/.github/workflows/periodic-test.yml b/.github/workflows/periodic-test.yml index 5bff88d28b..b640239a00 100644 --- a/.github/workflows/periodic-test.yml +++ b/.github/workflows/periodic-test.yml @@ -27,7 +27,7 @@ permissions: jobs: changes: name: Check - runs-on: ubuntu-22.04 + runs-on: blacksmith-4vcpu-ubuntu-2204 timeout-minutes: 15 strategy: fail-fast: false diff --git a/.github/workflows/pr-go-dependbot-cleanup.yml b/.github/workflows/pr-go-dependbot-cleanup.yml index 3c104bb93f..8b93d69bee 100644 --- a/.github/workflows/pr-go-dependbot-cleanup.yml +++ b/.github/workflows/pr-go-dependbot-cleanup.yml @@ -10,7 +10,7 @@ on: jobs: run: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 permissions: contents: write steps: diff --git a/.github/workflows/pr-no-generated-changes.yml b/.github/workflows/pr-no-generated-changes.yml index f4fd88d5aa..0af88dc8fd 100644 --- a/.github/workflows/pr-no-generated-changes.yml +++ b/.github/workflows/pr-no-generated-changes.yml @@ -20,7 +20,7 @@ permissions: jobs: run-check: - runs-on: ubuntu-24.04 + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - name: Generate GitHub App installation token id: app-token diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml index 4205b7f1cc..3b74e26a3b 100644 --- a/.github/workflows/pr-tests-arm64.yml +++ b/.github/workflows/pr-tests-arm64.yml @@ -8,7 +8,7 @@ permissions: jobs: cross-compile: name: Cross-compile all packages for ARM64 - runs-on: ubuntu-24.04 + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 steps: - name: Checkout repository @@ -39,7 +39,7 @@ jobs: arm64-unit-tests: name: ARM64 tests for ${{ matrix.package }} - runs-on: ubuntu-24.04-arm + runs-on: blacksmith-4vcpu-ubuntu-2404-arm timeout-minutes: 30 env: GIN_MODE: test diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 77898c5930..edc26c6136 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -8,7 +8,7 @@ permissions: jobs: run-tests: name: Run tests for ${{ matrix.package }} - runs-on: infra-tests + runs-on: blacksmith-4vcpu-ubuntu-2404 env: GIN_MODE: test strategy: @@ -93,7 +93,7 @@ jobs: validate-iac: name: Validate terraform - runs-on: ubuntu-24.04 + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout repository uses: actions/checkout@v5 diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 62bd96248c..a7849a1d2d 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -40,7 +40,7 @@ jobs: needs: - unit-tests - integration-tests - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 permissions: checks: write if: always() diff --git a/.github/workflows/push-main.yml b/.github/workflows/push-main.yml index bbbe715161..f05c299ade 100644 --- a/.github/workflows/push-main.yml +++ b/.github/workflows/push-main.yml @@ -25,7 +25,7 @@ jobs: needs: - unit-tests - integration-tests - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 permissions: checks: write if: always() diff --git a/.github/workflows/validate-openapi.yml b/.github/workflows/validate-openapi.yml index 279b7da535..8c26726e13 100644 --- a/.github/workflows/validate-openapi.yml +++ b/.github/workflows/validate-openapi.yml @@ -8,7 +8,7 @@ permissions: jobs: validate-openapi: name: Validate ${{ matrix.spec }} - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: matrix: spec: From 757e8176917bc4a5736d966e9e081acb48ec5a81 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Fri, 10 Apr 2026 11:23:00 +0200 Subject: [PATCH 02/10] Update integration_tests.yml Increased size --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index c1954ab669..e6616c3e17 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -9,7 +9,7 @@ on: required: true jobs: integration_tests: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 30 permissions: contents: read From 8c428b13d79fc8cbffae5068c7a59e5c96405810 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Fri, 10 Apr 2026 11:28:25 +0200 Subject: [PATCH 03/10] fix: skip modprobe nbd if built into kernel Blacksmith runners ship nbd as a kernel built-in, so modprobe fails with "module not found". Check /sys/module/nbd first and skip modprobe if nbd is already present. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/pr-tests-arm64.yml | 14 ++++++++++---- .github/workflows/pr-tests.yml | 8 ++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml index 3b74e26a3b..6a03360580 100644 --- a/.github/workflows/pr-tests-arm64.yml +++ b/.github/workflows/pr-tests-arm64.yml @@ -99,10 +99,16 @@ jobs: sudo mount -t hugetlbfs none /mnt/hugepages echo 256 | sudo tee /proc/sys/vm/nr_hugepages - # Install extra kernel modules (nbd is not in base modules on GitHub-hosted runners) - sudo apt-get update - sudo apt-get install -y linux-modules-extra-$(uname -r) - sudo modprobe nbd nbds_max=256 + # Enable NBD — skip modprobe if built into kernel + if [ -e /sys/module/nbd ]; then + echo "nbd is built into kernel, skipping modprobe" + else + sudo modprobe nbd nbds_max=256 2>/dev/null || { + echo "nbd not available, installing linux-modules-extra..." + sudo apt-get update && sudo apt-get install -y linux-modules-extra-$(uname -r) + sudo modprobe nbd nbds_max=256 + } + fi # Disable inotify watching of change events for NBD devices echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index edc26c6136..17f3120043 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -69,8 +69,12 @@ jobs: sudo mount -t hugetlbfs none /mnt/hugepages echo 2000 | sudo tee /proc/sys/vm/nr_hugepages - # Enable NBD - sudo modprobe nbd nbds_max=256 + # Enable NBD — skip modprobe if built into kernel + if [ -e /sys/module/nbd ]; then + echo "nbd is built into kernel, skipping modprobe" + else + sudo modprobe nbd nbds_max=256 + fi # Disable inotify watching of change events for NBD devices echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules From 63ec1a06c3691e3377c8690deb8f8d72a1738013 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Fri, 10 Apr 2026 11:29:45 +0200 Subject: [PATCH 04/10] fix: skip swapfile creation if already active Blacksmith runners may have a swapfile already in use. Check with swapon --show before attempting fallocate. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/actions/host-init/init-client.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/host-init/init-client.sh b/.github/actions/host-init/init-client.sh index 57c4946154..533cd3d09b 100644 --- a/.github/actions/host-init/init-client.sh +++ b/.github/actions/host-init/init-client.sh @@ -17,15 +17,15 @@ sudo mkdir -p /orchestrator/sandbox sudo mkdir -p /orchestrator/template sudo mkdir -p /orchestrator/build -# Add swapfile +# Add swapfile (skip if already active) SWAPFILE="/swapfile" -sudo fallocate -l 1G $SWAPFILE -sudo chmod 600 $SWAPFILE -sudo mkswap $SWAPFILE -sudo swapon $SWAPFILE - -# Make swapfile persistent -echo "$SWAPFILE none swap sw 0 0" | sudo tee -a /etc/fstab +if ! swapon --show | grep -q "$SWAPFILE"; then + sudo fallocate -l 1G $SWAPFILE + sudo chmod 600 $SWAPFILE + sudo mkswap $SWAPFILE + sudo swapon $SWAPFILE + echo "$SWAPFILE none swap sw 0 0" | sudo tee -a /etc/fstab +fi # Set swap settings sudo sysctl vm.swappiness=10 From 498223643f86223d02a0ac441899c0ccb36ae169 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Fri, 10 Apr 2026 11:33:44 +0200 Subject: [PATCH 05/10] fix: load nfs/nfsv3 kernel modules for NFS proxy e2e test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NFS proxy test mounts NFS inside a Docker container which requires nfs kernel modules on the host. Load nbd, nfs, nfsv3 — skip if built-in. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/pr-tests.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index d1103c581b..7cb4bd36ae 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -67,12 +67,14 @@ jobs: sudo mount -t hugetlbfs none /mnt/hugepages echo 2000 | sudo tee /proc/sys/vm/nr_hugepages - # Enable NBD — skip modprobe if built into kernel - if [ -e /sys/module/nbd ]; then - echo "nbd is built into kernel, skipping modprobe" - else - sudo modprobe nbd nbds_max=256 - fi + # Load kernel modules — skip if built into kernel + for mod in nbd nfs nfsv3; do + if [ -e /sys/module/${mod} ]; then + echo "${mod} is built into kernel" + else + sudo modprobe ${mod} 2>/dev/null && echo "${mod} loaded" || echo "WARNING: ${mod} not available" + fi + done # Disable inotify watching of change events for NBD devices echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules From c68ceaa449a5e11c23a54ca1dcce86d7ee543509 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Fri, 10 Apr 2026 11:34:31 +0200 Subject: [PATCH 06/10] fix: handle built-in kernel modules across all CI workflows Blacksmith runners have nbd/nfs built into kernel (6.5.13) and don't ship linux-modules-extra. Apply the same built-in check pattern to: - pr-tests-arm64.yml (remove apt-get install linux-modules-extra) - init-client.sh (integration tests) Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/actions/host-init/init-client.sh | 10 ++++++++-- .github/workflows/pr-tests-arm64.yml | 18 ++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/actions/host-init/init-client.sh b/.github/actions/host-init/init-client.sh index 533cd3d09b..595dcb4c3c 100644 --- a/.github/actions/host-init/init-client.sh +++ b/.github/actions/host-init/init-client.sh @@ -65,8 +65,14 @@ EOH sudo udevadm control --reload-rules sudo udevadm trigger -# Load the nbd module with 4096 devices -sudo modprobe nbd nbds_max=4096 +# Load kernel modules — skip if built into kernel +for mod in nbd nfs nfsv3; do + if [ -e /sys/module/${mod} ]; then + echo "${mod} is built into kernel" + else + sudo modprobe ${mod} 2>/dev/null && echo "${mod} loaded" || echo "WARNING: ${mod} not available" + fi +done # Create the directory for the fc mounts mkdir -p /fc-vm diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml index 6a03360580..7f65061c16 100644 --- a/.github/workflows/pr-tests-arm64.yml +++ b/.github/workflows/pr-tests-arm64.yml @@ -99,16 +99,14 @@ jobs: sudo mount -t hugetlbfs none /mnt/hugepages echo 256 | sudo tee /proc/sys/vm/nr_hugepages - # Enable NBD — skip modprobe if built into kernel - if [ -e /sys/module/nbd ]; then - echo "nbd is built into kernel, skipping modprobe" - else - sudo modprobe nbd nbds_max=256 2>/dev/null || { - echo "nbd not available, installing linux-modules-extra..." - sudo apt-get update && sudo apt-get install -y linux-modules-extra-$(uname -r) - sudo modprobe nbd nbds_max=256 - } - fi + # Load kernel modules — skip if built into kernel + for mod in nbd nfs nfsv3; do + if [ -e /sys/module/${mod} ]; then + echo "${mod} is built into kernel" + else + sudo modprobe ${mod} 2>/dev/null && echo "${mod} loaded" || echo "WARNING: ${mod} not available" + fi + done # Disable inotify watching of change events for NBD devices echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules From 2cb74b9676af4216bddae29f3f2342151c355687 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Fri, 10 Apr 2026 11:35:23 +0200 Subject: [PATCH 07/10] fix: restore nbds_max parameter for modprobe nbd NBD modprobe needs nbds_max parameter (4096 for integration tests, 256 for unit tests). Separate nbd from the generic module loop. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/actions/host-init/init-client.sh | 9 ++++++++- .github/workflows/pr-tests-arm64.yml | 9 ++++++++- .github/workflows/pr-tests.yml | 9 ++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/actions/host-init/init-client.sh b/.github/actions/host-init/init-client.sh index 595dcb4c3c..875abb8228 100644 --- a/.github/actions/host-init/init-client.sh +++ b/.github/actions/host-init/init-client.sh @@ -66,7 +66,7 @@ sudo udevadm control --reload-rules sudo udevadm trigger # Load kernel modules — skip if built into kernel -for mod in nbd nfs nfsv3; do +for mod in nfs nfsv3; do if [ -e /sys/module/${mod} ]; then echo "${mod} is built into kernel" else @@ -74,6 +74,13 @@ for mod in nbd nfs nfsv3; do fi done +# NBD needs nbds_max parameter +if [ -e /sys/module/nbd ]; then + echo "nbd is built into kernel" +else + sudo modprobe nbd nbds_max=4096 2>/dev/null && echo "nbd loaded" || echo "WARNING: nbd not available" +fi + # Create the directory for the fc mounts mkdir -p /fc-vm diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml index 7f65061c16..00471aeb1f 100644 --- a/.github/workflows/pr-tests-arm64.yml +++ b/.github/workflows/pr-tests-arm64.yml @@ -100,7 +100,7 @@ jobs: echo 256 | sudo tee /proc/sys/vm/nr_hugepages # Load kernel modules — skip if built into kernel - for mod in nbd nfs nfsv3; do + for mod in nfs nfsv3; do if [ -e /sys/module/${mod} ]; then echo "${mod} is built into kernel" else @@ -108,6 +108,13 @@ jobs: fi done + # NBD needs nbds_max parameter + if [ -e /sys/module/nbd ]; then + echo "nbd is built into kernel" + else + sudo modprobe nbd nbds_max=256 + fi + # Disable inotify watching of change events for NBD devices echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules sudo udevadm control --reload-rules diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 7cb4bd36ae..f0dade36fa 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -68,7 +68,7 @@ jobs: echo 2000 | sudo tee /proc/sys/vm/nr_hugepages # Load kernel modules — skip if built into kernel - for mod in nbd nfs nfsv3; do + for mod in nfs nfsv3; do if [ -e /sys/module/${mod} ]; then echo "${mod} is built into kernel" else @@ -76,6 +76,13 @@ jobs: fi done + # NBD needs nbds_max parameter + if [ -e /sys/module/nbd ]; then + echo "nbd is built into kernel" + else + sudo modprobe nbd nbds_max=256 + fi + # Disable inotify watching of change events for NBD devices echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules sudo udevadm control --reload-rules From 63d8e3016438ff512ecef7f8a1e15fdf25b28a01 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Fri, 10 Apr 2026 11:37:29 +0200 Subject: [PATCH 08/10] fix: make modprobe nbd non-fatal on ARM64 runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blacksmith ARM64 kernel (6.5.13) lacks nbd module. Make it non-fatal so CI proceeds — NBD-dependent tests will fail individually. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/pr-tests-arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml index 00471aeb1f..766aee203a 100644 --- a/.github/workflows/pr-tests-arm64.yml +++ b/.github/workflows/pr-tests-arm64.yml @@ -112,7 +112,7 @@ jobs: if [ -e /sys/module/nbd ]; then echo "nbd is built into kernel" else - sudo modprobe nbd nbds_max=256 + sudo modprobe nbd nbds_max=256 || echo "WARNING: nbd not available — NBD tests will fail" fi # Disable inotify watching of change events for NBD devices From 1532202d8e3867bc6903ac0cdad480b3138f6425 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Sun, 12 Apr 2026 14:47:40 +0200 Subject: [PATCH 09/10] fix: add legacy mount fallback for overlayfs on kernels < 6.8 The fsconfig/fsopen API requires kernel 6.8+. Blacksmith runners use kernel 6.5.13 which doesn't support it. Add fallback to traditional mount syscall with lowerdir= option (limited to ~4096 chars but sufficient for CI). Production nodes run kernel 6.8+ and continue using the fsconfig path. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../template/build/core/filesystem/ext4.go | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go b/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go index eca227e359..29bbfb8da9 100644 --- a/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go +++ b/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go @@ -276,8 +276,8 @@ func RemoveFile(ctx context.Context, rootfsPath string, filePath string) error { } // MountOverlayFS mounts an overlay filesystem with the specified layers at the given mount point. -// It requires kernel version 6.8 or later to use the fsconfig interface for overlayfs. -// Older mount syscall is not used because it has lowerdirs character limit (4096 characters). +// Uses the fsconfig interface (kernel 6.8+) which supports unlimited lowerdir entries. +// Falls back to the traditional mount syscall for older kernels (limited to ~4096 chars for lowerdirs). func MountOverlayFS(ctx context.Context, layers []string, mountPoint string) error { _, mountSpan := tracer.Start(ctx, "mount-overlay-fs", trace.WithAttributes( attribute.String("mount", mountPoint), @@ -285,34 +285,40 @@ func MountOverlayFS(ctx context.Context, layers []string, mountPoint string) err )) defer mountSpan.End() - // Open the filesystem for configuration + err := mountOverlayFsconfig(layers, mountPoint) + if err == nil { + return nil + } + + // Fallback to legacy mount for kernels < 6.8 + return mountOverlayLegacy(layers, mountPoint) +} + +// mountOverlayFsconfig uses the fsconfig/fsopen/fsmount API (kernel 6.8+). +// Supports unlimited lowerdir entries via repeated "lowerdir+" keys. +func mountOverlayFsconfig(layers []string, mountPoint string) error { fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC) if err != nil { return fmt.Errorf("fsopen failed: %w", err) } defer unix.Close(fsfd) - // Set lowerdir using FSCONFIG_SET_STRING for _, layer := range layers { - // https://docs.kernel.org/filesystems/overlayfs.html if err := unix.FsconfigSetString(fsfd, "lowerdir+", layer); err != nil { return fmt.Errorf("fsconfig lowerdir failed: %w", err) } } - // Finalize configuration if err := unix.FsconfigCreate(fsfd); err != nil { return fmt.Errorf("fsconfig create failed: %w", err) } - // Create the mount mfd, err := unix.Fsmount(fsfd, 0, 0) if err != nil { return fmt.Errorf("fsmount failed: %w", err) } defer unix.Close(mfd) - // Mount to target if err := unix.MoveMount(mfd, "", -1, mountPoint, unix.MOVE_MOUNT_F_EMPTY_PATH); err != nil { return fmt.Errorf("move mount failed: %w", err) } @@ -320,6 +326,19 @@ func MountOverlayFS(ctx context.Context, layers []string, mountPoint string) err return nil } +// mountOverlayLegacy uses the traditional mount syscall. +// Limited to ~4096 characters for the lowerdir option string. +func mountOverlayLegacy(layers []string, mountPoint string) error { + lowerdir := strings.Join(layers, ":") + + opts := fmt.Sprintf("lowerdir=%s", lowerdir) + if err := unix.Mount("overlay", mountPoint, "overlay", 0, opts); err != nil { + return fmt.Errorf("legacy overlay mount failed (opts length %d): %w", len(opts), err) + } + + return nil +} + // SetReservedBlocksOnHost sets the number of reserved filesystem blocks based on the desired reserved space in MB. // Reserved blocks are only usable by root (uid 0). func SetReservedBlocksOnHost(ctx context.Context, rootfsPath string, reservedSpaceMB int64, blockSize int64) error { From fd985e7c8217c9b305a1c256b65c5ca7d8046db3 Mon Sep 17 00:00:00 2001 From: e2b Date: Sun, 12 Apr 2026 14:48:24 +0200 Subject: [PATCH 10/10] Revert "fix: add legacy mount fallback for overlayfs on kernels < 6.8" This reverts commit 1532202d8e3867bc6903ac0cdad480b3138f6425. --- .../template/build/core/filesystem/ext4.go | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go b/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go index 29bbfb8da9..eca227e359 100644 --- a/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go +++ b/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go @@ -276,8 +276,8 @@ func RemoveFile(ctx context.Context, rootfsPath string, filePath string) error { } // MountOverlayFS mounts an overlay filesystem with the specified layers at the given mount point. -// Uses the fsconfig interface (kernel 6.8+) which supports unlimited lowerdir entries. -// Falls back to the traditional mount syscall for older kernels (limited to ~4096 chars for lowerdirs). +// It requires kernel version 6.8 or later to use the fsconfig interface for overlayfs. +// Older mount syscall is not used because it has lowerdirs character limit (4096 characters). func MountOverlayFS(ctx context.Context, layers []string, mountPoint string) error { _, mountSpan := tracer.Start(ctx, "mount-overlay-fs", trace.WithAttributes( attribute.String("mount", mountPoint), @@ -285,40 +285,34 @@ func MountOverlayFS(ctx context.Context, layers []string, mountPoint string) err )) defer mountSpan.End() - err := mountOverlayFsconfig(layers, mountPoint) - if err == nil { - return nil - } - - // Fallback to legacy mount for kernels < 6.8 - return mountOverlayLegacy(layers, mountPoint) -} - -// mountOverlayFsconfig uses the fsconfig/fsopen/fsmount API (kernel 6.8+). -// Supports unlimited lowerdir entries via repeated "lowerdir+" keys. -func mountOverlayFsconfig(layers []string, mountPoint string) error { + // Open the filesystem for configuration fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC) if err != nil { return fmt.Errorf("fsopen failed: %w", err) } defer unix.Close(fsfd) + // Set lowerdir using FSCONFIG_SET_STRING for _, layer := range layers { + // https://docs.kernel.org/filesystems/overlayfs.html if err := unix.FsconfigSetString(fsfd, "lowerdir+", layer); err != nil { return fmt.Errorf("fsconfig lowerdir failed: %w", err) } } + // Finalize configuration if err := unix.FsconfigCreate(fsfd); err != nil { return fmt.Errorf("fsconfig create failed: %w", err) } + // Create the mount mfd, err := unix.Fsmount(fsfd, 0, 0) if err != nil { return fmt.Errorf("fsmount failed: %w", err) } defer unix.Close(mfd) + // Mount to target if err := unix.MoveMount(mfd, "", -1, mountPoint, unix.MOVE_MOUNT_F_EMPTY_PATH); err != nil { return fmt.Errorf("move mount failed: %w", err) } @@ -326,19 +320,6 @@ func mountOverlayFsconfig(layers []string, mountPoint string) error { return nil } -// mountOverlayLegacy uses the traditional mount syscall. -// Limited to ~4096 characters for the lowerdir option string. -func mountOverlayLegacy(layers []string, mountPoint string) error { - lowerdir := strings.Join(layers, ":") - - opts := fmt.Sprintf("lowerdir=%s", lowerdir) - if err := unix.Mount("overlay", mountPoint, "overlay", 0, opts); err != nil { - return fmt.Errorf("legacy overlay mount failed (opts length %d): %w", len(opts), err) - } - - return nil -} - // SetReservedBlocksOnHost sets the number of reserved filesystem blocks based on the desired reserved space in MB. // Reserved blocks are only usable by root (uid 0). func SetReservedBlocksOnHost(ctx context.Context, rootfsPath string, reservedSpaceMB int64, blockSize int64) error {