diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 701fd4d..bfbf0a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,13 +33,15 @@ jobs: - name: Install gRPC + Protobuf run: brew install grpc - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 + - name: Cache Rust build artifacts + uses: actions/cache@v4 with: - workspaces: compute/tokenizers-sys + path: compute/tokenizers-sys/target + key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }} + restore-keys: tokenizers-sys-${{ runner.os }}- + + - name: Build tokenizers-sys + run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml - name: Cache CMake fetched dependencies uses: actions/cache@v4 @@ -80,13 +82,15 @@ jobs: - name: Install gRPC + Protobuf run: brew install grpc - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 + - name: Cache Rust build artifacts + uses: actions/cache@v4 with: - workspaces: compute/tokenizers-sys + path: compute/tokenizers-sys/target + key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }} + restore-keys: tokenizers-sys-${{ runner.os }}- + + - name: Build tokenizers-sys + run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml - name: Cache CMake fetched dependencies uses: actions/cache@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4793888..fc8934a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,13 +33,15 @@ jobs: - name: Install gRPC + Protobuf + coreutils run: brew install grpc coreutils - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 + - name: Cache Rust build artifacts + uses: actions/cache@v4 with: - workspaces: compute/tokenizers-sys + path: compute/tokenizers-sys/target + key: tokenizers-sys-${{ runner.os }}-${{ hashFiles('compute/tokenizers-sys/Cargo.lock', 'compute/tokenizers-sys/src/**') }} + restore-keys: tokenizers-sys-${{ runner.os }}- + + - name: Build tokenizers-sys + run: cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml - name: Cache CMake fetched dependencies uses: actions/cache@v4 diff --git a/Makefile b/Makefile index f65fa5d..c85b563 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,12 @@ CMAKE_BUILD := cmake --build $(BUILD_DIR) -j$(JOBS) # ── Helpers ────────────────────────────────────────────────────────────────── +.PHONY: _tokenizers_sys +_tokenizers_sys: ## Build Rust tokenizers-sys static lib (required before cmake) + @cargo build --release --manifest-path compute/tokenizers-sys/Cargo.toml + .PHONY: _configure -_configure: +_configure: _tokenizers_sys @if [ ! -f "$(BUILD_DIR)/CMakeCache.txt" ]; then \ echo "==> Configuring ($(BUILD_TYPE))"; \ $(CMAKE_CONFIGURE); \ diff --git a/compute/CMakeLists.txt b/compute/CMakeLists.txt index d64318a..22c5aab 100644 --- a/compute/CMakeLists.txt +++ b/compute/CMakeLists.txt @@ -2,17 +2,14 @@ cmake_minimum_required(VERSION 3.16) project(compute_backend VERSION 1.0.0 LANGUAGES CXX) -# ── Rust / Corrosion ────────────────────────────────────────────────────────── -# Corrosion drives cargo build from CMake; tokenizers_sys is the Rust staticlib -# that wraps HuggingFace tokenizers 0.32.1. -include(FetchContent) -FetchContent_Declare( - Corrosion - GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git - GIT_TAG v0.5.1 +# ── Rust / tokenizers-sys ───────────────────────────────────────────────────── +# libtokenizers_sys.a is built by `cargo build --release` in compute/tokenizers-sys/ +# BEFORE cmake runs (same pattern as `brew install grpc` before cmake). +# CMake just links the pre-built static library — no Corrosion, no rustup needed. +add_library(tokenizers_sys STATIC IMPORTED GLOBAL) +set_target_properties(tokenizers_sys PROPERTIES + IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/tokenizers-sys/target/release/libtokenizers_sys.a" ) -FetchContent_MakeAvailable(Corrosion) -corrosion_import_crate(MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/tokenizers-sys/Cargo.toml) set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON)