fix: replace Corrosion with pre-built cargo artifact (Neurons-ofq)

.github/workflows/build.yml

@@ -33,13 +33,15 @@ jobs:
       - name: Install gRPC + Protobuf
         run: brew install grpc
 
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@v2
+      - name: Cache Rust build artifacts
+        uses: actions/cache@v4
         with:
-          workspaces: compute/tokenizers-sys
+          path: compute/tokenizers-sys/target
+          key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }}
+          restore-keys: tokenizers-sys-${{ runner.os }}-
+
+      - name: Build tokenizers-sys
+        run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml
 
       - name: Cache CMake fetched dependencies
         uses: actions/cache@v4
@@ -80,13 +82,15 @@ jobs:
       - name: Install gRPC + Protobuf
         run: brew install grpc
 
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@v2
+      - name: Cache Rust build artifacts
+        uses: actions/cache@v4
         with:
-          workspaces: compute/tokenizers-sys
+          path: compute/tokenizers-sys/target
+          key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }}
+          restore-keys: tokenizers-sys-${{ runner.os }}-
+
+      - name: Build tokenizers-sys
+        run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml
 
       - name: Cache CMake fetched dependencies
         uses: actions/cache@v4

.github/workflows/test.yml

@@ -33,13 +33,15 @@ jobs:
       - name: Install gRPC + Protobuf + coreutils
         run: brew install grpc coreutils
 
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Cache Rust dependencies
-        uses: Swatinem/rust-cache@v2
+      - name: Cache Rust build artifacts
+        uses: actions/cache@v4
         with:
-          workspaces: compute/tokenizers-sys
+          path: compute/tokenizers-sys/target
+          key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }}
+          restore-keys: tokenizers-sys-${{ runner.os }}-
+
+      - name: Build tokenizers-sys
+        run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml
 
       - name: Cache CMake fetched dependencies
         uses: actions/cache@v4

Makefile

@@ -21,8 +21,12 @@ CMAKE_BUILD     := cmake --build $(BUILD_DIR) -j$(JOBS)
 
 # ── Helpers ──────────────────────────────────────────────────────────────────
 
+.PHONY: _tokenizers_sys
+_tokenizers_sys: ## Build Rust tokenizers-sys static lib (required before cmake)
+	@cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml
+
 .PHONY: _configure
-_configure:
+_configure: _tokenizers_sys
 	@if [ ! -f "$(BUILD_DIR)/CMakeCache.txt" ]; then \
 	    echo "==> Configuring ($(BUILD_TYPE))"; \
 	    $(CMAKE_CONFIGURE); \

compute/CMakeLists.txt

@@ -2,17 +2,14 @@ cmake_minimum_required(VERSION 3.16)
 
 project(compute_backend VERSION 1.0.0 LANGUAGES CXX)
 
-# ── Rust / Corrosion ──────────────────────────────────────────────────────────
-# Corrosion drives cargo build from CMake; tokenizers_sys is the Rust staticlib
-# that wraps HuggingFace tokenizers 0.32.1.
-include(FetchContent)
-FetchContent_Declare(
-    Corrosion
-    GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
-    GIT_TAG        v0.5.1
+# ── Rust / tokenizers-sys ─────────────────────────────────────────────────────
+# libtokenizers_sys.a is built by `cargo build --release` in compute/tokenizers-sys/
+# BEFORE cmake runs (same pattern as `brew install grpc` before cmake).
+# CMake just links the pre-built static library — no Corrosion, no rustup needed.
+add_library(tokenizers_sys STATIC IMPORTED GLOBAL)
+set_target_properties(tokenizers_sys PROPERTIES
+    IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/tokenizers-sys/target/release/libtokenizers_sys.a"
 )
-FetchContent_MakeAvailable(Corrosion)
-corrosion_import_crate(MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/tokenizers-sys/Cargo.toml)
 
 set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)