From 1bee73015c352e5636d71bfea6c9bbf075a69037 Mon Sep 17 00:00:00 2001 From: Dex Date: Thu, 14 May 2026 21:56:37 -0400 Subject: [PATCH] fix: replace Corrosion with pre-built cargo artifact (Neurons-ofq) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrosion was calling rustup show/toolchain during cmake configure, which failed on the macos-26 CI runner where the rustup binary is the bootstrap installer rather than the toolchain manager. Same pattern as gRPC: build libtokenizers_sys.a via a plain 'cargo build --release' step BEFORE cmake runs. CMake links the pre-built static library directly — no Corrosion, no rustup needed. - Remove Corrosion FetchContent from compute/CMakeLists.txt - Add IMPORTED STATIC target pointing at tokenizers-sys/target/release/ - Replace dtolnay/rust-toolchain + Swatinem/rust-cache with a single 'cargo build' step + artifact cache in build.yml and test.yml - Wire _tokenizers_sys as a prerequisite of _configure in Makefile --- .github/workflows/build.yml | 28 ++++++++++++++++------------ .github/workflows/test.yml | 14 ++++++++------ Makefile | 6 +++++- compute/CMakeLists.txt | 17 +++++++---------- 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 701fd4d..bfbf0a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,13 +33,15 @@ jobs: - name: Install gRPC + Protobuf run: brew install grpc - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 + - name: Cache Rust build artifacts + uses: actions/cache@v4 with: - workspaces: compute/tokenizers-sys + path: compute/tokenizers-sys/target + key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }} + restore-keys: tokenizers-sys-${{ runner.os }}- + + - name: Build tokenizers-sys + run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml - name: Cache CMake fetched dependencies uses: actions/cache@v4 @@ -80,13 +82,15 @@ jobs: - name: Install gRPC + Protobuf run: brew install grpc - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 + - name: Cache Rust build artifacts + uses: actions/cache@v4 with: - workspaces: compute/tokenizers-sys + path: compute/tokenizers-sys/target + key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }} + restore-keys: tokenizers-sys-${{ runner.os }}- + + - name: Build tokenizers-sys + run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml - name: Cache CMake fetched dependencies uses: actions/cache@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4793888..fc8934a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,13 +33,15 @@ jobs: - name: Install gRPC + Protobuf + coreutils run: brew install grpc coreutils - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 + - name: Cache Rust build artifacts + uses: actions/cache@v4 with: - workspaces: compute/tokenizers-sys + path: compute/tokenizers-sys/target + key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }} + restore-keys: tokenizers-sys-${{ runner.os }}- + + - name: Build tokenizers-sys + run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml - name: Cache CMake fetched dependencies uses: actions/cache@v4 diff --git a/Makefile b/Makefile index f65fa5d..c85b563 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,12 @@ CMAKE_BUILD := cmake --build $(BUILD_DIR) -j$(JOBS) # ── Helpers ────────────────────────────────────────────────────────────────── +.PHONY: _tokenizers_sys +_tokenizers_sys: ## Build Rust tokenizers-sys static lib (required before cmake) + @cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml + .PHONY: _configure -_configure: +_configure: _tokenizers_sys @if [ ! -f "$(BUILD_DIR)/CMakeCache.txt" ]; then \ echo "==> Configuring ($(BUILD_TYPE))"; \ $(CMAKE_CONFIGURE); \ diff --git a/compute/CMakeLists.txt b/compute/CMakeLists.txt index d64318a..22c5aab 100644 --- a/compute/CMakeLists.txt +++ b/compute/CMakeLists.txt @@ -2,17 +2,14 @@ cmake_minimum_required(VERSION 3.16) project(compute_backend VERSION 1.0.0 LANGUAGES CXX) -# ── Rust / Corrosion ────────────────────────────────────────────────────────── -# Corrosion drives cargo build from CMake; tokenizers_sys is the Rust staticlib -# that wraps HuggingFace tokenizers 0.32.1. -include(FetchContent) -FetchContent_Declare( - Corrosion - GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git - GIT_TAG v0.5.1 +# ── Rust / tokenizers-sys ───────────────────────────────────────────────────── +# libtokenizers_sys.a is built by `cargo build --release` in compute/tokenizers-sys/ +# BEFORE cmake runs (same pattern as `brew install grpc` before cmake). +# CMake just links the pre-built static library — no Corrosion, no rustup needed. +add_library(tokenizers_sys STATIC IMPORTED GLOBAL) +set_target_properties(tokenizers_sys PROPERTIES + IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/tokenizers-sys/target/release/libtokenizers_sys.a" ) -FetchContent_MakeAvailable(Corrosion) -corrosion_import_crate(MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/tokenizers-sys/Cargo.toml) set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON)