diff --git a/csharp/.gitignore b/csharp/.gitignore new file mode 100644 index 0000000..7563d84 --- /dev/null +++ b/csharp/.gitignore @@ -0,0 +1,8 @@ +/.vs/ +/local_feed/ +/Microsoft.ML.OnnxRuntime.EP.TensorRT/runtimes/**/native/*.dll +*.nupkg +*.snupkg +**/packages/* +!**/packages/build/ +**/bin/* diff --git a/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj b/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj new file mode 100644 index 0000000..2169e0d --- /dev/null +++ b/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj @@ -0,0 +1,51 @@ + + + + net9.0 + enable + enable + + win-x64 + win-arm64 + + + Microsoft.ML.OnnxRuntime.EP.TensorRT + 1.0.0 + ORT + Microsoft + A package for a TensorRT plugin EP. + readme.md + + + MIT + https://github.com/microsoft/onnxruntime-inference-examples + git + + + true + snupkg + + True + + + + + + + + + + + + + + + + + diff --git a/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/TensorRTEp.cs b/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/TensorRTEp.cs new file mode 100644 index 0000000..5ab8a3b --- /dev/null +++ b/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/TensorRTEp.cs @@ -0,0 +1,83 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime.EP.TensorRT; + +public static class TensorRTEp +{ + /// + /// Returns the path to the plugin EP library DLL contained by this package. + /// Can be passed to OrtEnv::RegisterExecutionProviderLibrary(). + /// + /// Note: It is recommended that plugin EP packages provide this information to applications. + /// + /// EP library path + /// If the EP DLL file path does not exist + public static string GetLibraryPath() + { + string rootDir = GetNativeDirectory(); + string osArch = $"{GetOSTag()}-{GetArchTag()}"; + string epDllPath = Path.GetFullPath(Path.Combine(rootDir, "runtimes", osArch, + "native", "tensorrt_plugin_ep.dll")); + + if (!File.Exists(epDllPath)) + { + // This indicates a packaging error. + throw new FileNotFoundException($"Did not find EP DLL file: {epDllPath}"); + } + + return epDllPath; + } + + /// + /// Returns the names of the EPs created by the plugin EP library. + /// Can be used to select a OrtEpDevice from those returned by OrtEnv::GetEpDevices(). + /// + /// Note: It is recommended that plugin EP packages provide this information to applications. + /// + /// Array of EP names + public static string[] GetEpNames() + { + return ["TensorRTPluginExecutionProvider"]; + } + + /// + /// Returns the name of the one EP supported by this plugin EP library. + /// + /// Note: This is a convenience function exposed by plugin EP packages that only have one EP name. + /// + /// + public static string GetEpName() + { + return GetEpNames()[0]; + } + + private static string GetNativeDirectory() + { + var assemblyDir = Path.GetDirectoryName(typeof(TensorRTEp).Assembly.Location); + + // Try returning where this assembly lives (works for framework-dependent) + if (!string.IsNullOrEmpty(assemblyDir) && Directory.Exists(assemblyDir)) + return assemblyDir; + + // Fallback to AppContext.BaseDirectory (works for single-file/self-contained) + return AppContext.BaseDirectory; + } + + private static string GetOSTag() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return "win"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "linux"; + return "unknown"; + } + + private static string GetArchTag() + { + return RuntimeInformation.OSArchitecture switch + { + Architecture.X64 => "x64", + Architecture.Arm64 => "arm64", + _ => "unknown" + }; + } +} diff --git a/csharp/SampleApp/Program.cs b/csharp/SampleApp/Program.cs new file mode 100644 index 0000000..28d3e77 --- /dev/null +++ b/csharp/SampleApp/Program.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.ML.OnnxRuntime.EP.TensorRT; +using Microsoft.ML.OnnxRuntime; + +class Program +{ + static void Main() + { + string epLibPath = TensorRTEp.GetLibraryPath(); + string epName = TensorRTEp.GetEpName(); + string epRegistrationName = epName; + + var env = OrtEnv.Instance(); + env.RegisterExecutionProviderLibrary(epRegistrationName, epLibPath); + Console.WriteLine($"Registered EP library: {epLibPath}"); + + try + { + // Find the OrtEpDevice for the EP + OrtEpDevice? epDevice = null; + foreach (var d in env.GetEpDevices()) + { + if (string.Equals(epName, d.EpName, StringComparison.OrdinalIgnoreCase)) + { + epDevice = d; + } + } + + if (epDevice == null) + { + Console.Error.WriteLine($"ERROR: Unable to find OrtEpDevice with name {epName}"); + return; + } + Console.WriteLine($"Found OrtEpDevice for EP: {epName}"); + + // Create session with EP + using var sessionOptions = new SessionOptions(); + sessionOptions.AppendExecutionProvider(env, [epDevice], new Dictionary { }); + sessionOptions.AddSessionConfigEntry("session.disable_cpu_ep_fallback", "1"); // Don't run on CPU EP + + string inputModelPath = Path.Combine(AppContext.BaseDirectory, "mul.onnx"); + Console.WriteLine($"Loading model: {inputModelPath}"); + + using var session = new InferenceSession(inputModelPath, sessionOptions); + + // Run model + float[] inputData = [1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f]; + using var inputOrtValue = OrtValue.CreateTensorValueFromMemory(inputData, [2, 3]); + var inputValues = new List { inputOrtValue, inputOrtValue }.AsReadOnly(); + var inputNames = new List { "x", "y" }.AsReadOnly(); + using var runOptions = new RunOptions(); + + using var outputs = session.Run(runOptions, inputNames, inputValues, session.OutputNames); + + Console.WriteLine($"Input: {string.Join(", ", inputData)}"); + Console.WriteLine($"Output: {string.Join(", ", outputs[0].GetTensorDataAsSpan().ToArray())}"); + } + finally + { + env.UnregisterExecutionProviderLibrary(epRegistrationName); + } + } +} diff --git a/csharp/SampleApp/SampleApp.csproj b/csharp/SampleApp/SampleApp.csproj new file mode 100644 index 0000000..6ce8a95 --- /dev/null +++ b/csharp/SampleApp/SampleApp.csproj @@ -0,0 +1,26 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + PreserveNewest + + + + + + + diff --git a/csharp/SampleApp/mul.onnx b/csharp/SampleApp/mul.onnx new file mode 100644 index 0000000..f7a8b1d Binary files /dev/null and b/csharp/SampleApp/mul.onnx differ diff --git a/csharp/nuget.config b/csharp/nuget.config new file mode 100644 index 0000000..5954963 --- /dev/null +++ b/csharp/nuget.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/csharp/readme.md b/csharp/readme.md new file mode 100644 index 0000000..87d67cc --- /dev/null +++ b/csharp/readme.md @@ -0,0 +1,34 @@ +# TensorRT Plugin Execution Provider with C# + +## Contents +- `Microsoft.ML.OnnxRuntime.EP.TensorRT/`: Contains files for the TensorRT plugin EP C# NuGet package. `TensorRTEp.cs` provides helper functions to get the EP library path and the EP name. +- `SampleApp/`: Contains a sample C# application showing example usage of the TensorRT plugin EP C# NuGet package. +- `setup.bat`: Batch script to generate the NuGet package. + +## Build Instructions +This example currently only supports Windows x64 and Windows ARM64. + +### Build the native plugin EP library + +Follow instructions [here](../README.md#build-instructions) to build the native library. + +### Build the C\# NuGet Package + +Set the environment variable `TENSORRT_PLUGIN_EP_LIBRARY_PATH` to the path to the native plugin EP shared library. E.g., `tensorrt_plugin_ep.dll`. + +Run `setup.bat` from this directory. Pass the build configuration (e.g., Release or Debug) as an argument. + +``` +.\setup.bat Release +``` + +The generated NuGet package will be copied to the `./local_feed` directory. + +## Build and run the sample application + +Build and run the sample application. + +``` +dotnet build .\SampleApp\SampleApp.csproj -c Release +dotnet run --project .\SampleApp\SampleApp.csproj -c Release +``` diff --git a/csharp/setup.bat b/csharp/setup.bat new file mode 100644 index 0000000..14289cf --- /dev/null +++ b/csharp/setup.bat @@ -0,0 +1,63 @@ +@echo off + +REM Builds NuGet package that wraps TensorRT plugin EP + +IF "%~1"=="" ( + echo ERROR: No build configuration specified. + echo Usage: .\setup.bat [Debug^|Release] + exit /b 1 +) + +SET "BUILD_CONFIG=%~1" + +if "%TENSORRT_PLUGIN_EP_LIBRARY_PATH%"=="" ( + echo ERROR: TENSORRT_PLUGIN_EP_LIBRARY_PATH environment variable is not set. + exit /b 1 +) + +if not exist "%TENSORRT_PLUGIN_EP_LIBRARY_PATH%" ( + echo ERROR: EP library "%TENSORRT_PLUGIN_EP_LIBRARY_PATH%" not found. + exit /b 1 +) + +set "ARCH=%PROCESSOR_ARCHITECTURE%" +if defined PROCESSOR_ARCHITEW6432 set "ARCH=%PROCESSOR_ARCHITEW6432%" + +if /i "%ARCH%"=="AMD64" ( + set "DEST_EP_DLL_FOLDER=.\Microsoft.ML.OnnxRuntime.EP.TensorRT\runtimes\win-x64\native\" +) else if /i "%ARCH%"=="ARM64" ( + set "DEST_EP_DLL_FOLDER=.\Microsoft.ML.OnnxRuntime.EP.TensorRT\runtimes\win-arm64\native\" +) else ( + echo ERROR: Unknown architecture "%ARCH%" + exit /b 1 +) + +if not exist "%DEST_EP_DLL_FOLDER%" ( + mkdir "%DEST_EP_DLL_FOLDER%" || ( + echo ERROR: Failed to create "%DEST_EP_DLL_FOLDER%". + exit /b 1 + ) +) + +echo Copying EP DLL to "%DEST_EP_DLL_FOLDER%" +copy /Y "%TENSORRT_PLUGIN_EP_LIBRARY_PATH%" "%DEST_EP_DLL_FOLDER%" >nul + +if errorlevel 1 ( + echo ERROR: Failed to copy EP library to "%DEST_EP_DLL_FOLDER%". + exit /b 1 +) + +echo Building NuGet package ("%BUILD_CONFIG%") ... +dotnet build .\Microsoft.ML.OnnxRuntime.EP.TensorRT\Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj -c "%BUILD_CONFIG%" +dotnet pack .\Microsoft.ML.OnnxRuntime.EP.TensorRT\Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj -c "%BUILD_CONFIG%" + +set "LOCAL_FEED_FOLDER=local_feed" +if not exist "%LOCAL_FEED_FOLDER%" ( + mkdir "%LOCAL_FEED_FOLDER%" || ( + echo ERROR: Failed to create "%LOCAL_FEED_FOLDER%" + exit /b 1 + ) +) + +copy /Y .\Microsoft.ML.OnnxRuntime.EP.TensorRT\bin\"%BUILD_CONFIG%"\Microsoft.ML.OnnxRuntime.EP.TensorRT.*.nupkg .\local_feed\ +copy /Y .\Microsoft.ML.OnnxRuntime.EP.TensorRT\bin\"%BUILD_CONFIG%"\Microsoft.ML.OnnxRuntime.EP.TensorRT.*.snupkg .\local_feed\