diff --git a/sdk/cpp/CMakeLists.txt b/sdk/cpp/CMakeLists.txt index 17b11e07..01402484 100644 --- a/sdk/cpp/CMakeLists.txt +++ b/sdk/cpp/CMakeLists.txt @@ -74,6 +74,69 @@ if (UNIX) target_link_libraries(CppSdk PUBLIC ${CMAKE_DL_LIBS}) endif() +# ----------------------------- +# Native dependencies (NuGet) — Windows only +# Auto-download Core DLL and ONNX Runtime DLLs needed at runtime. +# Versions are kept in sync with deps_versions.json. +# ----------------------------- +if(WIN32) + +set(FL_CORE_VERSION "1.1.0" CACHE STRING "Microsoft.AI.Foundry.Local.Core NuGet version") +set(FL_ORT_VERSION "1.25.1" CACHE STRING "Microsoft.ML.OnnxRuntime.Foundry NuGet version") +set(FL_ORTGENAI_VERSION "0.13.2" CACHE STRING "Microsoft.ML.OnnxRuntimeGenAI.Foundry NuGet version") +set(FL_NATIVE_DEPS_DIR "${CMAKE_CURRENT_BINARY_DIR}/_native_deps") + +function(fl_ensure_nuget_package PKG_NAME PKG_VERSION) + set(PKG_DIR "${FL_NATIVE_DEPS_DIR}/${PKG_NAME}.${PKG_VERSION}") + if(NOT EXISTS "${PKG_DIR}") + message(STATUS "Downloading ${PKG_NAME} ${PKG_VERSION}...") + find_program(NUGET_EXE nuget) + if(NOT NUGET_EXE) + message(FATAL_ERROR "nuget.exe not found on PATH. Install NuGet CLI: https://www.nuget.org/downloads") + endif() + execute_process( + COMMAND ${NUGET_EXE} install ${PKG_NAME} -Version ${PKG_VERSION} + -OutputDirectory ${FL_NATIVE_DEPS_DIR} -NonInteractive + RESULT_VARIABLE result + ) + if(NOT result EQUAL 0) + message(FATAL_ERROR "Failed to download ${PKG_NAME} ${PKG_VERSION}") + endif() + endif() +endfunction() + +fl_ensure_nuget_package("Microsoft.AI.Foundry.Local.Core" ${FL_CORE_VERSION}) +fl_ensure_nuget_package("Microsoft.ML.OnnxRuntime.Foundry" ${FL_ORT_VERSION}) +fl_ensure_nuget_package("Microsoft.ML.OnnxRuntimeGenAI.Foundry" ${FL_ORTGENAI_VERSION}) + +set(FL_CORE_DLL_DIR "${FL_NATIVE_DEPS_DIR}/Microsoft.AI.Foundry.Local.Core.${FL_CORE_VERSION}/runtimes/win-x64/native") +set(FL_ORT_DLL_DIR "${FL_NATIVE_DEPS_DIR}/Microsoft.ML.OnnxRuntime.Foundry.${FL_ORT_VERSION}/runtimes/win-x64/native") +set(FL_ORTGENAI_DLL_DIR "${FL_NATIVE_DEPS_DIR}/Microsoft.ML.OnnxRuntimeGenAI.Foundry.${FL_ORTGENAI_VERSION}/runtimes/win-x64/native") + +# Helper function: copy runtime DLLs next to any target that links CppSdk +function(fl_copy_runtime_dlls TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${FL_CORE_DLL_DIR}/Microsoft.AI.Foundry.Local.Core.dll" + $ + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${FL_ORT_DLL_DIR}" + $ + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${FL_ORTGENAI_DLL_DIR}" + $ + COMMENT "Copying Foundry Local runtime DLLs for ${TARGET_NAME}..." + ) +endfunction() + +else() # Non-Windows: provide a no-op fl_copy_runtime_dlls + +function(fl_copy_runtime_dlls TARGET_NAME) + # No DLL copying needed on non-Windows platforms +endfunction() + +endif() # WIN32 + # ----------------------------- # Sample executable # ----------------------------- @@ -83,6 +146,32 @@ add_executable(CppSdkSample target_link_libraries(CppSdkSample PRIVATE CppSdk) +# Copy DLLs for the SDK sample +fl_copy_runtime_dlls(CppSdkSample) + +# ----------------------------- +# Vision sample (Responses API) — built if present +# ----------------------------- +set(VISION_SAMPLE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/sample/web-server-responses-vision") + +if(EXISTS "${VISION_SAMPLE_DIR}/main.cpp") + find_package(CURL QUIET) + if(CURL_FOUND) + add_executable(WebServerResponsesVision + ${VISION_SAMPLE_DIR}/main.cpp + ${VISION_SAMPLE_DIR}/stb_impl.cpp + ) + + target_link_libraries(WebServerResponsesVision PRIVATE CppSdk CURL::libcurl) + fl_copy_runtime_dlls(WebServerResponsesVision) + message(STATUS "Vision sample: enabled") + else() + message(STATUS "Vision sample: disabled (curl not found — add 'curl' to vcpkg.json)") + endif() +else() + message(STATUS "Vision sample: not found") +endif() + # ----------------------------- # Unit tests # ----------------------------- @@ -153,3 +242,40 @@ endif() # Make Visual Studio start/debug this target by default set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT CppSdkSample) + +# ----------------------------- +# Install — produces the redistributable SDK zip layout: +# lib/CppSdk.lib +# include/... +# bin/*.dll (runtime DLLs) +# cmake/FoundryLocalConfig.cmake +# ----------------------------- +install(TARGETS CppSdk + ARCHIVE DESTINATION $,lib/debug,lib> +) + +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ + DESTINATION include + FILES_MATCHING PATTERN "*.h" +) + +# Install runtime DLLs into bin/ (Windows only) +if(WIN32) + install(DIRECTORY "${FL_CORE_DLL_DIR}/" DESTINATION bin FILES_MATCHING PATTERN "*.dll") + install(DIRECTORY "${FL_ORT_DLL_DIR}/" DESTINATION bin FILES_MATCHING PATTERN "*.dll") + install(DIRECTORY "${FL_ORTGENAI_DLL_DIR}/" DESTINATION bin FILES_MATCHING PATTERN "*.dll") +endif() + +# Install CMake config file +install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/FoundryLocalConfig.cmake DESTINATION cmake) + +# Install vision sample into the zip +if(EXISTS "${VISION_SAMPLE_DIR}/main.cpp") + install(DIRECTORY "${VISION_SAMPLE_DIR}/" + DESTINATION sample/web-server-responses-vision + PATTERN "build" EXCLUDE + PATTERN "out" EXCLUDE + PATTERN "_native_deps" EXCLUDE + PATTERN "vcpkg_installed" EXCLUDE + ) +endif() diff --git a/sdk/cpp/README.md b/sdk/cpp/README.md index a8b35534..017f2fa6 100644 --- a/sdk/cpp/README.md +++ b/sdk/cpp/README.md @@ -11,10 +11,11 @@ The Foundry Local C++ SDK provides a C++17 static library for running AI models - **Chat completions** — synchronous and streaming via OpenAI-compatible types - **Audio transcription** — transcribe audio files with streaming support - **Tool calling** — define tools and handle tool-call responses in chat completions -- **Execution providers** — discover, download, and register EPs with per-EP progress reporting - **Download progress** — wire up a callback for real-time download percentage - **Model variants** — select specific hardware/quantization variants per model alias - **Optional web service** — start an OpenAI-compatible REST endpoint +- **Execution providers** — discover, download, and register EPs with per-EP progress reporting +- **Auto NuGet download** — CMake auto-downloads native runtime DLLs at configure time ## Prerequisites @@ -24,6 +25,7 @@ The Foundry Local C++ SDK provides a C++17 static library for running AI models | **Ninja** | Ships with Visual Studio 2022 | | **vcpkg** | Set the `VCPKG_ROOT` environment variable to your vcpkg installation | | **MSVC** (or clang-cl) | Visual Studio 2022 Build Tools or full IDE | +| **NuGet CLI** | Required for auto-downloading native runtime DLLs. Install via `winget install Microsoft.NuGet` | ## Building from Source @@ -58,24 +60,14 @@ This uses the `x64-debug` preset which: - Uses the **Ninja** generator - Resolves C++ dependencies via **vcpkg** (`nlohmann-json`, `ms-gsl`, `gtest`) - Builds with the `x64-windows-static-md` triplet +- Auto-downloads native runtime DLLs via **NuGet**: + - `Microsoft.AI.Foundry.Local.Core` (1.1.0) — Foundry Local core runtime + - `Microsoft.ML.OnnxRuntime.Foundry` (1.25.1) — ONNX Runtime + - `Microsoft.ML.OnnxRuntimeGenAI.Foundry` (0.13.2) — ONNX Runtime GenAI -### 3. Obtain runtime DLLs - -The SDK loads `Microsoft.AI.Foundry.Local.Core.dll` at runtime from the executable's directory. Download the required DLLs via NuGet and copy them next to the built executable: - -```powershell -nuget install Microsoft.AI.Foundry.Local.Core -Version 1.1.0 -OutputDirectory _native_deps -nuget install Microsoft.ML.OnnxRuntime.Foundry -Version 1.25.1 -OutputDirectory _native_deps -nuget install Microsoft.ML.OnnxRuntimeGenAI.Foundry -Version 0.13.2 -OutputDirectory _native_deps +NuGet packages are cached in `out/build//_native_deps/` and only downloaded on first configure. Runtime DLLs are automatically copied next to executables via post-build steps. -Copy-Item _native_deps\Microsoft.AI.Foundry.Local.Core.1.1.0\runtimes\win-x64\native\*.dll out\build\x64-debug\ -Copy-Item _native_deps\Microsoft.ML.OnnxRuntime.Foundry.1.25.1\runtimes\win-x64\native\*.dll out\build\x64-debug\ -Copy-Item _native_deps\Microsoft.ML.OnnxRuntimeGenAI.Foundry.0.13.2\runtimes\win-x64\native\*.dll out\build\x64-debug\ -``` - -> **Note:** This step is only needed once (or when upgrading versions). The DLLs are not re-downloaded if already present. - -### 4. Build +### 3. Build ```bash cmake --build --preset x64-debug @@ -128,6 +120,29 @@ int main() { } ``` +### Vision Sample (Responses API) + +A complete vision sample is included at `sample/web-server-responses-vision/`. It demonstrates image understanding using the Responses API with streaming via cURL. + +Build and run from the SDK root: + +```bash +cmake --preset x64-debug +cmake --build --preset x64-debug --target WebServerResponsesVision +.\out\build\x64-debug\WebServerResponsesVision.exe qwen3.5-0.8b +``` + +Or build standalone from the sample directory: + +```bash +cd sample/web-server-responses-vision +cmake --preset x64-debug +cmake --build --preset x64-debug +.\out\build\x64-debug\web-server-responses-vision.exe qwen3.5-0.8b +``` + +See [sample/web-server-responses-vision/README.md](sample/web-server-responses-vision/README.md) for full details. + ## Usage ### Initialization @@ -317,6 +332,90 @@ if (result.success) { } ``` +### Using the Prebuilt SDK (Zip) + +#### Creating the zip + +Build and install the SDK to produce the redistributable layout: + +```bash +cd sdk/cpp + +# Release build +cmake --preset x64-release +cmake --build --preset x64-release +cmake --install out/build/x64-release --prefix out/foundry-local-cpp-sdk + +# Optional: also install Debug lib for consumers who need Debug builds +cmake --preset x64-debug +cmake --build --preset x64-debug +cmake --install out/build/x64-debug --prefix out/foundry-local-cpp-sdk +``` + +This creates: + +``` +out/foundry-local-cpp-sdk/ +├── include/ # Public headers +├── lib/CppSdk.lib # Prebuilt static library +├── bin/ # Runtime DLLs (Core, OnnxRuntime, OnnxRuntimeGenAI) +├── cmake/ # FoundryLocalConfig.cmake +└── README.md +``` + +Zip the `out/foundry-local-cpp-sdk/` folder and distribute. + +#### Using the zip in your project + +1. Unzip to a folder (e.g. `foundry-local-cpp-sdk/`) +2. In your `CMakeLists.txt`: + +```cmake +cmake_minimum_required(VERSION 3.20) +project(my-app) + +set(CMAKE_CXX_STANDARD 17) +set(VCPKG_TARGET_TRIPLET "x64-windows-static-md" CACHE STRING "") + +list(APPEND CMAKE_PREFIX_PATH "${CMAKE_CURRENT_SOURCE_DIR}/foundry-local-cpp-sdk") +find_package(FoundryLocal REQUIRED) + +add_executable(my-app main.cpp) +target_link_libraries(my-app PRIVATE FoundryLocal::FoundryLocal) + +# Auto-copies Core DLL, ORT DLLs next to the exe +fl_copy_runtime_dlls(my-app) +``` + +3. Create a `vcpkg.json` with the required transitive dependencies: + +```json +{ + "dependencies": ["nlohmann-json", "ms-gsl"] +} +``` + +4. Build: + +```bash +cmake -G Ninja -B build -DCMAKE_TOOLCHAIN_FILE="%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake" +cmake --build build +``` + +> **Note:** Match your build type to the SDK's. If the zip only contains a Release lib, build your project in Release (`-DCMAKE_BUILD_TYPE=Release`) to avoid MSVC runtime-library mismatches. If both Debug and Release libs are included, CMake selects the correct one automatically. + +### Using the SDK from Source + +Include the SDK via `add_subdirectory` (e.g. from the repo): + +```cmake +add_subdirectory(path/to/sdk/cpp ${CMAKE_CURRENT_BINARY_DIR}/CppSdk) + +add_executable(my_app main.cpp) +target_link_libraries(my_app PRIVATE CppSdk) +fl_copy_runtime_dlls(my_app) +``` + ## Configuration | Property | Type | Default | Description | @@ -384,7 +483,8 @@ sdk/cpp/ │ └── tool_types.h # Tool calling types ├── src/ # Private implementation ├── sample/ -│ └── main.cpp # Sample application +│ ├── main.cpp # Sample application +│ └── web-server-responses-vision/ # Vision sample (Responses API) ├── test/ # Unit & E2E tests (GTest) ├── CMakeLists.txt ├── CMakePresets.json @@ -396,10 +496,12 @@ sdk/cpp/ | Error | Cause | Fix | |---|---|---| -| `Failed to load shared library: Microsoft.AI.Foundry.Local.Core.dll` | Runtime DLLs not next to executable | Copy DLLs from NuGet packages (see step 3 in Building from Source) | | `DML provider requested, but GenAI has not been built with DML support` | GPU variant selected but ONNX Runtime GenAI lacks DML | Select a CPU variant or update Foundry Local | -| `OgaGenerator_TokenCount not found in onnxruntime-genai` | Version mismatch between Foundry Local components | Re-download NuGet packages with matching versions | -| `API version [N] is not available` | ONNX Runtime version too old for the Foundry Local service | Re-download NuGet packages with matching versions | +| `OgaGenerator_TokenCount not found in onnxruntime-genai` | Version mismatch between Foundry Local components | Update NuGet package versions in CMakeLists.txt | +| `API version [N] is not available` | ONNX Runtime version too old for the Foundry Local service | Update NuGet package versions in CMakeLists.txt | +| `nuget.exe not found on PATH` | NuGet CLI not installed | Install via `winget install Microsoft.NuGet` | +| `Failed to load shared library: Microsoft.AI.Foundry.Local.Core.dll` | Runtime DLLs not next to executable | Reconfigure with `cmake --preset x64-debug` to re-download NuGet packages, then rebuild | +| NuGet packages not installed or DLLs not copied correctly | Stale or corrupted build cache | Delete the `out` folder (`rmdir /s /q out`) and reconfigure from scratch: `cmake --preset x64-debug && cmake --build --preset x64-debug` | ## License diff --git a/sdk/cpp/cmake/FoundryLocalConfig.cmake b/sdk/cpp/cmake/FoundryLocalConfig.cmake new file mode 100644 index 00000000..fe1fe84e --- /dev/null +++ b/sdk/cpp/cmake/FoundryLocalConfig.cmake @@ -0,0 +1,88 @@ +# FoundryLocalConfig.cmake +# +# Imported target: FoundryLocal::FoundryLocal +# +# Usage in your CMakeLists.txt: +# list(APPEND CMAKE_PREFIX_PATH "") +# find_package(FoundryLocal REQUIRED) +# target_link_libraries(my_app PRIVATE FoundryLocal::FoundryLocal) +# fl_copy_runtime_dlls(my_app) + +get_filename_component(_FL_SDK_DIR "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE) + +# Validate SDK layout +if(NOT EXISTS "${_FL_SDK_DIR}/include/foundry_local.h") + message(FATAL_ERROR + "FoundryLocal SDK incomplete: include/foundry_local.h not found at ${_FL_SDK_DIR}/include/. " + "Ensure CMAKE_PREFIX_PATH points to the correct SDK directory." + ) +endif() + +if(NOT EXISTS "${_FL_SDK_DIR}/lib/CppSdk.lib" AND NOT EXISTS "${_FL_SDK_DIR}/lib/debug/CppSdk.lib") + message(FATAL_ERROR + "FoundryLocal SDK incomplete: CppSdk.lib not found at ${_FL_SDK_DIR}/lib/. " + "Build and install the SDK first: cmake --install out/build/x64-release --prefix " + ) +endif() + +if(WIN32 AND NOT EXISTS "${_FL_SDK_DIR}/bin") + message(FATAL_ERROR + "FoundryLocal SDK incomplete: bin/ directory not found at ${_FL_SDK_DIR}/bin/. " + "Runtime DLLs are required. Rebuild and install the SDK." + ) +endif() + +# Create imported static library target +if(NOT TARGET FoundryLocal::FoundryLocal) + add_library(FoundryLocal::FoundryLocal STATIC IMPORTED) + + # Support both Debug and Release libs if available + if(EXISTS "${_FL_SDK_DIR}/lib/debug/CppSdk.lib") + set_target_properties(FoundryLocal::FoundryLocal PROPERTIES + IMPORTED_LOCATION_RELEASE "${_FL_SDK_DIR}/lib/CppSdk.lib" + IMPORTED_LOCATION_DEBUG "${_FL_SDK_DIR}/lib/debug/CppSdk.lib" + INTERFACE_INCLUDE_DIRECTORIES "${_FL_SDK_DIR}/include" + ) + elseif(EXISTS "${_FL_SDK_DIR}/lib/CppSdk.lib") + # Single-config: warn if consumer build type doesn't match + if(CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE STREQUAL "Release") + message(WARNING + "FoundryLocal SDK was built in Release mode. " + "Linking with CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} may cause " + "MSVC runtime-library mismatches. Consider using Release or " + "rebuilding the SDK in Debug mode." + ) + endif() + set_target_properties(FoundryLocal::FoundryLocal PROPERTIES + IMPORTED_LOCATION "${_FL_SDK_DIR}/lib/CppSdk.lib" + INTERFACE_INCLUDE_DIRECTORIES "${_FL_SDK_DIR}/include" + ) + else() + message(FATAL_ERROR "FoundryLocal SDK library not found at ${_FL_SDK_DIR}/lib/") + endif() + + # Require nlohmann_json and GSL from vcpkg (consumer must have these) + find_package(nlohmann_json CONFIG REQUIRED) + find_package(Microsoft.GSL CONFIG REQUIRED) + + set_property(TARGET FoundryLocal::FoundryLocal APPEND PROPERTY + INTERFACE_LINK_LIBRARIES + nlohmann_json::nlohmann_json + Microsoft.GSL::GSL + ) +endif() + +# Runtime DLLs directory +set(FL_RUNTIME_DLL_DIR "${_FL_SDK_DIR}/bin") + +# Helper function: copy Foundry Local runtime DLLs next to an executable +function(fl_copy_runtime_dlls TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${FL_RUNTIME_DLL_DIR}" + $ + COMMENT "Copying Foundry Local runtime DLLs for ${TARGET_NAME}..." + ) +endfunction() + +set(FoundryLocal_FOUND TRUE) diff --git a/sdk/cpp/sample/main.cpp b/sdk/cpp/sample/main.cpp index 1812956d..7c377da9 100644 --- a/sdk/cpp/sample/main.cpp +++ b/sdk/cpp/sample/main.cpp @@ -3,7 +3,7 @@ #include "foundry_local.h" -#include +#include #include #include #include @@ -118,7 +118,7 @@ void ChatNonStreaming(Manager& manager, const std::string& alias) { PreferCpuVariant(*concreteModel); } - model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; return true; }); + model->Download([](float pct) { printf("\rDownloading: %5.1f%%", pct); fflush(stdout); return true; }); std::cout << "\n"; model->Load(); @@ -211,7 +211,7 @@ void TranscribeAudio(Manager& manager, const std::string& alias, const std::stri PreferCpuVariant(*concreteModel); } - model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; return true; }); + model->Download([](float pct) { printf("\rDownloading: %5.1f%%", pct); fflush(stdout); return true; }); std::cout << "\n"; model->Load(); @@ -263,7 +263,7 @@ void ChatWithToolCalling(Manager& manager, const std::string& alias) { PreferCpuVariant(*concreteModel); } - model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; return true; }); + model->Download([](float pct) { printf("\rDownloading: %5.1f%%", pct); fflush(stdout); return true; }); std::cout << "\n"; model->Load(); @@ -397,9 +397,8 @@ int main(int argc, char* argv[]) { if (!currentEp.empty()) std::cout << "\n"; currentEp = epName; } - std::cout << "\r " << std::left << std::setw(30) << epName - << " " << std::right << std::fixed << std::setprecision(1) - << std::setw(6) << percent << "% " << std::flush; + printf("\r %-30s %5.1f%%", epName.c_str(), percent); + fflush(stdout); }); if (!currentEp.empty()) std::cout << "\n"; } else { diff --git a/sdk/cpp/sample/web-server-responses-vision/CMakeLists.txt b/sdk/cpp/sample/web-server-responses-vision/CMakeLists.txt new file mode 100644 index 00000000..b6dbcbd0 --- /dev/null +++ b/sdk/cpp/sample/web-server-responses-vision/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.20) +project(web-server-responses-vision LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Match the SDK's vcpkg triplet +set(VCPKG_TARGET_TRIPLET "x64-windows-static-md" CACHE STRING "") + +# Try find_package first (prebuilt SDK zip), fall back to add_subdirectory (repo) +find_package(FoundryLocal QUIET) +if(NOT FoundryLocal_FOUND) + message(STATUS "Prebuilt FoundryLocal not found — building SDK from source") + set(FOUNDRY_SDK_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..") + set(BUILD_TESTING OFF CACHE BOOL "" FORCE) + add_subdirectory(${FOUNDRY_SDK_DIR} ${CMAKE_CURRENT_BINARY_DIR}/CppSdk) + set(FL_SDK_TARGET CppSdk) +else() + message(STATUS "Using prebuilt FoundryLocal SDK") + set(FL_SDK_TARGET FoundryLocal::FoundryLocal) +endif() + +find_package(CURL REQUIRED) + +add_executable(web-server-responses-vision main.cpp stb_impl.cpp) + +target_link_libraries(web-server-responses-vision PRIVATE + ${FL_SDK_TARGET} + CURL::libcurl +) + +# Copy runtime DLLs next to the executable +fl_copy_runtime_dlls(web-server-responses-vision) diff --git a/sdk/cpp/sample/web-server-responses-vision/CMakePresets.json b/sdk/cpp/sample/web-server-responses-vision/CMakePresets.json new file mode 100644 index 00000000..1ec183fe --- /dev/null +++ b/sdk/cpp/sample/web-server-responses-vision/CMakePresets.json @@ -0,0 +1,31 @@ +{ + "version": 6, + "configurePresets": [ + { + "name": "x64-debug", + "displayName": "MSVC x64 Debug", + "inherits": [], + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake", + "cacheVariables": { + "CMAKE_C_COMPILER": "cl.exe", + "CMAKE_CXX_COMPILER": "cl.exe", + "CMAKE_BUILD_TYPE": "Debug", + "VCPKG_TARGET_TRIPLET": "x64-windows-static-md", + "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/../../triplets" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + } + ], + "buildPresets": [ + { + "name": "x64-debug", + "configurePreset": "x64-debug" + } + ] +} diff --git a/sdk/cpp/sample/web-server-responses-vision/README.md b/sdk/cpp/sample/web-server-responses-vision/README.md new file mode 100644 index 00000000..cf9f343d --- /dev/null +++ b/sdk/cpp/sample/web-server-responses-vision/README.md @@ -0,0 +1,84 @@ +# Foundry Local C++ Vision Sample (Responses API) + +This sample demonstrates vision (image understanding) capabilities using the Foundry Local web service and the OpenAI Responses API. + +> **Windows-only** — requires MSVC or clang-cl (MSVC-compatible toolchain). + +## Features + +- **Vision inference** — send an image to a vision-capable model and get a description +- **Streaming** — token-by-token output via Server-Sent Events (SSE) +- **Responses API** — uses the `/v1/responses` endpoint (not chat completions) +- Uses a default test image (`test_image.jpg`) + +## Prerequisites + +| Requirement | Notes | +|---|---| +| **CMake >= 3.20** | Ships with Visual Studio 2022 | +| **Ninja** | Ships with Visual Studio 2022 | +| **vcpkg** | Set the `VCPKG_ROOT` environment variable to your vcpkg installation | +| **MSVC** (or clang-cl) | Visual Studio 2022 Build Tools or full IDE | +| **NuGet CLI** | Must be on PATH. Install via `winget install Microsoft.NuGet` | + +The sample downloads the specified model the first time it runs (skips if already cached). + +## Build + +Open an **x64 Native Tools Command Prompt for VS 2022** (or run `vcvars64.bat`), then navigate to the sample directory: + +```bash +cd sdk/cpp/sample/web-server-responses-vision +``` + +### Configure (CMake + vcpkg) + +```bash +cmake --preset x64-debug +``` + +### Build + +```bash +cmake --build --preset x64-debug +``` + +CMake will automatically: +- Install vcpkg dependencies (`nlohmann-json`, `ms-gsl`, `curl`, `stb`) +- Download the required NuGet packages (`Microsoft.AI.Foundry.Local.Core`, `Microsoft.ML.OnnxRuntime.Foundry`, `Microsoft.ML.OnnxRuntimeGenAI.Foundry`) +- Copy runtime DLLs next to the executable after build + +The built executable will be at `out/build/x64-debug/web-server-responses-vision.exe`. + +## Run the sample + +```bash +.\out\build\x64-debug\web-server-responses-vision.exe qwen3.5-0.8b +``` + +The sample starts the local web service, sends vision requests via the Responses API to `http://localhost:/v1`, prints the model output, and then stops the web service. + +## How it works + +1. **Initialize** — creates the `Manager` singleton with web service configuration +2. **Execution providers** — discovers and downloads compatible EPs via `DiscoverEps()` and `DownloadAndRegisterEps()` +3. **Model setup** — resolves the model alias, downloads if not cached, and loads into memory +4. **Web service** — starts the local Foundry web service on a random port +5. **Image encoding** — loads the image via stb, resizes to max 512px (preserving aspect ratio), and base64-encodes as JPEG +6. **Vision request** — builds the Responses API request body with `input_text` + `input_image` content parts +7. **Streaming** — sends the request via cURL with SSE streaming, printing tokens as they arrive +8. **Cleanup** — stops the web service, unloads the model, and destroys the manager + +## Troubleshooting + +| Error | Cause | Fix | +|---|---|---| +| `Failed to load image: ` | Default image not found | Ensure `test_image.jpg` is present next to the source file | +| `Model 'xyz' not found in catalog` | Invalid model alias | Check available models printed in the error output | +| `WebGPU execution provider is not supported` | WebGPU EP not available in this OnnxRuntime build | Ensure `DownloadAndRegisterEps()` runs before model load to install the EP | +| cURL connection refused | Web service failed to start | Ensure `config.web` is set and no port conflicts exist | +| NuGet packages not installed or DLLs not copied correctly | Stale or corrupted build cache | Delete the `out` folder (`rmdir /s /q out`) and reconfigure from scratch: `cmake --preset x64-debug && cmake --build --preset x64-debug` | + +## License + +Licensed under the MIT License. diff --git a/sdk/cpp/sample/web-server-responses-vision/main.cpp b/sdk/cpp/sample/web-server-responses-vision/main.cpp new file mode 100644 index 00000000..cd13db15 --- /dev/null +++ b/sdk/cpp/sample/web-server-responses-vision/main.cpp @@ -0,0 +1,355 @@ +// +// +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "foundry_local.h" +// + +#ifdef _WIN32 +#include +#endif + +using json = nlohmann::json; + +// ─── Base64 encoding ──────────────────────────────────────────────────────── + +static const char kBase64Chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +std::string Base64Encode(const std::vector& data) { + std::string out; + out.reserve(((data.size() + 2) / 3) * 4); + size_t i = 0; + while (i < data.size()) { + uint32_t octet_a = i < data.size() ? data[i++] : 0; + uint32_t octet_b = i < data.size() ? data[i++] : 0; + uint32_t octet_c = i < data.size() ? data[i++] : 0; + uint32_t triple = (octet_a << 16) | (octet_b << 8) | octet_c; + out.push_back(kBase64Chars[(triple >> 18) & 0x3F]); + out.push_back(kBase64Chars[(triple >> 12) & 0x3F]); + out.push_back(kBase64Chars[(triple >> 6) & 0x3F]); + out.push_back(kBase64Chars[triple & 0x3F]); + } + size_t mod = data.size() % 3; + if (mod == 1) { + out[out.size() - 2] = '='; + out[out.size() - 1] = '='; + } else if (mod == 2) { + out[out.size() - 1] = '='; + } + return out; +} + +// Load and resize a local image, returning (base64_str, media_type). +// Mirrors the Python sample's resize_and_encode(path, max_dim=512). +std::pair ResizeAndEncode(const std::filesystem::path& path, int maxDim = 512) { + int w = 0, h = 0, channels = 0; + unsigned char* img = stbi_load(path.string().c_str(), &w, &h, &channels, 3); + if (!img) { + throw std::runtime_error("Failed to load image: " + path.string()); + } + + int newW = w, newH = h; + if ((std::max)(w, h) > maxDim) { + if (w >= h) { + newW = maxDim; + newH = static_cast(static_cast(h) * maxDim / w); + } else { + newH = maxDim; + newW = static_cast(static_cast(w) * maxDim / h); + } + // Clamp to at least 1 pixel for extreme aspect ratios + newW = (std::max)(newW, 1); + newH = (std::max)(newH, 1); + + const auto resizedSize = + static_cast::size_type>(newW) * + static_cast::size_type>(newH) * + static_cast::size_type>(3); + std::vector resized(resizedSize); + unsigned char* result = stbir_resize_uint8_linear( + img, w, h, 0, resized.data(), newW, newH, 0, STBIR_RGB); + stbi_image_free(img); + if (!result) { + throw std::runtime_error("Failed to resize image"); + } + + std::cout << " (resized to " << newW << "x" << newH << ")" << std::endl; + + // Encode resized image to JPEG in memory + std::vector jpegBuf; + int writeOk = stbi_write_jpg_to_func( + [](void* ctx, void* data, int size) { + auto* buf = static_cast*>(ctx); + auto* bytes = static_cast(data); + buf->insert(buf->end(), bytes, bytes + size); + }, + &jpegBuf, newW, newH, 3, resized.data(), 90); + if (!writeOk) { + throw std::runtime_error("Failed to encode resized image to JPEG"); + } + + return {Base64Encode(jpegBuf), "image/jpeg"}; + } + + // No resize needed — encode original to JPEG + std::vector jpegBuf; + int writeOk = stbi_write_jpg_to_func( + [](void* ctx, void* data, int size) { + auto* buf = static_cast*>(ctx); + auto* bytes = static_cast(data); + buf->insert(buf->end(), bytes, bytes + size); + }, + &jpegBuf, w, h, 3, img, 90); + stbi_image_free(img); + if (!writeOk) { + throw std::runtime_error("Failed to encode image to JPEG"); + } + + return {Base64Encode(jpegBuf), "image/jpeg"}; +} + +// Persistent buffer for SSE parsing across cURL callbacks. +struct SseBuffer { + std::string partial; // incomplete line carried over between callbacks + bool done = false; // set when [DONE] is received +}; + +// Process a single complete SSE line. Returns true if [DONE] was received. +static bool ProcessSseLine(const std::string& line) { + if (line.rfind("data: ", 0) != 0) return false; + std::string data = line.substr(6); + if (data == "[DONE]") return true; + + try { + auto j = json::parse(data); + std::string type = j.value("type", ""); + if (type == "response.output_text.delta") { + std::string delta = j.value("delta", ""); + std::cout << delta << std::flush; + } + } catch (...) { + // Skip malformed JSON + } + return false; +} + +// cURL SSE streaming callback — appends to a persistent buffer and +// processes only complete lines, retaining any trailing partial line. +// Returns 0 to abort the transfer once [DONE] is observed. +static size_t StreamWriteCallback(char* ptr, size_t size, size_t nmemb, + void* userdata) { + size_t totalBytes = size * nmemb; + auto* buf = static_cast(userdata); + + if (buf->done) return 0; // abort transfer + + buf->partial.append(ptr, totalBytes); + + // Process all complete lines (terminated by \n) + std::string::size_type pos = 0; + std::string::size_type newline; + while ((newline = buf->partial.find('\n', pos)) != std::string::npos) { + std::string line = buf->partial.substr(pos, newline - pos); + // Strip trailing \r + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + if (ProcessSseLine(line)) { + buf->done = true; + buf->partial.clear(); + return 0; // abort transfer cleanly + } + pos = newline + 1; + } + // Retain any trailing partial line for the next callback + buf->partial.erase(0, pos); + + return totalBytes; +} + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cout << "Usage: web-server-responses-vision " << std::endl; + std::cout << " Example: web-server-responses-vision qwen3.5-0.8b" << std::endl; + return 1; + } + + const std::string modelAlias = argv[1]; + const std::filesystem::path imagePath = + std::filesystem::path(__FILE__).parent_path() / "test_image.jpg"; + + CURLcode globalRes = curl_global_init(CURL_GLOBAL_DEFAULT); + if (globalRes != CURLE_OK) { + std::cerr << "Error: curl_global_init failed: " << curl_easy_strerror(globalRes) << std::endl; + return 1; + } + + try { + // + foundry_local::Configuration config("foundry_local_samples"); + config.web = foundry_local::WebServiceConfig{}; + + foundry_local::Manager::Create(config); + auto& manager = foundry_local::Manager::Instance(); + + // Discover and download execution providers + try { + auto eps = manager.DiscoverEps(); + std::cout << "\nAvailable execution providers:" << std::endl; + for (const auto& ep : eps) { + std::cout << " " << ep.name << std::endl; + } + + if (!eps.empty()) { + std::cout << "\nDownloading execution providers:" << std::endl; + std::string currentEp; + manager.DownloadAndRegisterEps([&](const std::string& epName, double percent) { + if (epName != currentEp) { + if (!currentEp.empty()) std::cout << std::endl; + currentEp = epName; + } + // Matches Python: print(f"\r {ep_name:<30} {percent:5.1f}%", ...) + printf("\r %-30s %5.1f%%", epName.c_str(), percent); + fflush(stdout); + }); + if (!currentEp.empty()) std::cout << std::endl; + } else { + std::cout << "\nNo execution providers to download." << std::endl; + } + } catch (const std::exception& ex) { + std::cerr << "EP discovery/download skipped: " << ex.what() << std::endl; + } + // + + // + auto& catalog = manager.GetCatalog(); + auto* model = catalog.GetModel(modelAlias); + if (!model) { + auto models = catalog.GetModels(); + std::string available; + for (auto* m : models) { + available += " " + m->GetAlias(); + } + throw std::runtime_error( + "Model '" + modelAlias + "' not found in catalog. Available:" + available); + } + + if (!model->IsCached()) { + std::cout << "\nDownloading model " << modelAlias << "..." << std::endl; + model->Download([](float pct) { + printf("\rDownloading model: %5.1f%%", pct); + fflush(stdout); + return true; + }); + std::cout << "\nModel downloaded" << std::endl; + } + + std::cout << "\nLoading model..." << std::endl; + model->Load(); + std::cout << "Model loaded" << std::endl; + // + + // + std::cout << "\nStarting web service..." << std::endl; + manager.StartWebService(); + auto endpoints = manager.GetWebServiceEndpoints(); + if (endpoints.empty()) { + throw std::runtime_error("No web service endpoints available"); + } + std::string baseUrl = endpoints[0]; + if (!baseUrl.empty() && baseUrl.back() == '/') { + baseUrl.pop_back(); + } + baseUrl += "/v1"; + std::cout << "Web service started" << std::endl; + + // Use cURL to call the local Foundry web service Responses API + // (C++ equivalent of OpenAI SDK used in the Python sample) + std::string responsesUrl = baseUrl + "/responses"; + // + + // + std::cout << "\nPreparing image: " << imagePath.string() << std::endl; + auto [imageB64, mediaType] = ResizeAndEncode(imagePath); + + json visionInput = json::array({ + { + {"type", "message"}, + {"role", "user"}, + {"content", json::array({ + {{"type", "input_text"}, {"text", "Describe this image."}}, + { + {"type", "input_image"}, + {"image_data", imageB64}, + {"media_type", mediaType}, + } + })} + } + }); + + std::cout << "\nStreaming vision response..." << std::endl; + + json requestBody = { + {"model", model->GetId()}, + {"input", visionInput}, + {"stream", true}, + }; + std::string body = requestBody.dump(); + + CURL* curl = curl_easy_init(); + if (!curl) { + throw std::runtime_error("Failed to initialize cURL"); + } + + struct curl_slist* headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + headers = curl_slist_append(headers, "Accept: text/event-stream"); + + curl_easy_setopt(curl, CURLOPT_URL, responsesUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, StreamWriteCallback); + + SseBuffer sseBuf; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &sseBuf); + + std::cout << "[ASSISTANT]: " << std::flush; + CURLcode res = curl_easy_perform(curl); + std::cout << std::endl; + + if (res != CURLE_OK && !(res == CURLE_WRITE_ERROR && sseBuf.done)) { + std::cerr << "cURL error: " << curl_easy_strerror(res) << std::endl; + } + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + // + + manager.StopWebService(); + model->Unload(); + foundry_local::Manager::Destroy(); + + } catch (const std::exception& ex) { + std::cerr << "Error: " << ex.what() << std::endl; + curl_global_cleanup(); + return 1; + } + + curl_global_cleanup(); + return 0; +} diff --git a/sdk/cpp/sample/web-server-responses-vision/stb_impl.cpp b/sdk/cpp/sample/web-server-responses-vision/stb_impl.cpp new file mode 100644 index 00000000..ae81fc3b --- /dev/null +++ b/sdk/cpp/sample/web-server-responses-vision/stb_impl.cpp @@ -0,0 +1,8 @@ +// stb implementation file — include exactly once per project. +#define STB_IMAGE_IMPLEMENTATION +#define STB_IMAGE_RESIZE2_IMPLEMENTATION +#define STB_IMAGE_WRITE_IMPLEMENTATION + +#include +#include +#include diff --git a/sdk/cpp/sample/web-server-responses-vision/test_image.jpg b/sdk/cpp/sample/web-server-responses-vision/test_image.jpg new file mode 100644 index 00000000..73a4e800 Binary files /dev/null and b/sdk/cpp/sample/web-server-responses-vision/test_image.jpg differ diff --git a/sdk/cpp/sample/web-server-responses-vision/vcpkg-configuration.json b/sdk/cpp/sample/web-server-responses-vision/vcpkg-configuration.json new file mode 100644 index 00000000..a5253fb7 --- /dev/null +++ b/sdk/cpp/sample/web-server-responses-vision/vcpkg-configuration.json @@ -0,0 +1,6 @@ +{ + "default-registry": { + "kind": "builtin", + "baseline": "a9f0cd0345fb29cd227d802f1fd1917c28f8e5a3" + } +} diff --git a/sdk/cpp/sample/web-server-responses-vision/vcpkg.json b/sdk/cpp/sample/web-server-responses-vision/vcpkg.json new file mode 100644 index 00000000..7d7d593e --- /dev/null +++ b/sdk/cpp/sample/web-server-responses-vision/vcpkg.json @@ -0,0 +1,10 @@ +{ + "name": "web-server-responses-vision", + "version-string": "0.1.0", + "dependencies": [ + "nlohmann-json", + "ms-gsl", + "curl", + "stb" + ] +} diff --git a/sdk/cpp/vcpkg.json b/sdk/cpp/vcpkg.json index 459a72c1..6845f8ee 100644 --- a/sdk/cpp/vcpkg.json +++ b/sdk/cpp/vcpkg.json @@ -4,6 +4,8 @@ "dependencies": [ "nlohmann-json", "ms-gsl", - "gtest" + "gtest", + "curl", + "stb" ] }