Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Directory.Build.targets
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@
</PropertyGroup>

<!-- Target to verify the version in the built assembly matches the expected git version -->
<Target Name="VerifyAssemblyVersion" AfterTargets="Build" Condition="'$(MSBuildProjectName)' == 'Braintrust.Sdk' or '$(MSBuildProjectName)' == 'Braintrust.Sdk.OpenAI' or '$(MSBuildProjectName)' == 'Braintrust.Sdk.Anthropic'">
<Target Name="VerifyAssemblyVersion" AfterTargets="Build" Condition="'$(SkipVersionVerification)' != 'true' and ('$(MSBuildProjectName)' == 'Braintrust.Sdk' or '$(MSBuildProjectName)' == 'Braintrust.Sdk.OpenAI' or '$(MSBuildProjectName)' == 'Braintrust.Sdk.Anthropic')">
<Message Text="Verifying SDK version in built assembly..." Importance="high" />

<!-- Get the expected version (recompute it to ensure consistency) -->
Expand Down
19 changes: 15 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ This library provides tools for **evaluating** and **tracing** AI applications i

- **Evaluate** your AI models with custom test cases and scoring functions
- **Trace** LLM calls and monitor AI application performance with OpenTelemetry
- **Integrate** seamlessly with OpenAI, Anthropic, Microsoft Agent Framework, and other LLM providers
- **Integrate** seamlessly with OpenAI, Anthropic, Microsoft Agent Framework, Microsoft.Extensions.AI (IChatClient), and other LLM providers

This SDK is currently in BETA status and APIs may change.

Expand All @@ -35,20 +35,31 @@ dotnet add package Braintrust.Sdk.OpenAI
dotnet add package Braintrust.Sdk.Anthropic
```

### Microsoft.Extensions.AI integration (IChatClient)

```bash
dotnet add package Braintrust.Sdk.Extensions.AI
```

Works with any `IChatClient` provider: OpenAI, Azure OpenAI, Ollama, etc.

### Microsoft Agent Framework integration

```bash
dotnet add package Braintrust.Sdk.AgentFramework
```

For agent orchestration with `ChatClientAgent`. Includes IChatClient + agent-level tracing.

### Or add to your .csproj file

```xml
<ItemGroup>
<PackageReference Include="Braintrust.Sdk" Version="version goes here" />
<PackageReference Include="Braintrust.Sdk.OpenAI" Version="version goes here" /> <!-- optional -->
<PackageReference Include="Braintrust.Sdk.Anthropic" Version="version goes here" /> <!-- optional -->
<PackageReference Include="Braintrust.Sdk.AgentFramework" Version="version goes here" /> <!-- optional -->
<PackageReference Include="Braintrust.Sdk.Extensions.AI" Version="version goes here" /> <!-- IChatClient -->
<PackageReference Include="Braintrust.Sdk.OpenAI" Version="version goes here" /> <!-- raw OpenAI SDK -->
<PackageReference Include="Braintrust.Sdk.Anthropic" Version="version goes here" /> <!-- raw Anthropic SDK -->
<PackageReference Include="Braintrust.Sdk.AgentFramework" Version="version goes here" /> <!-- Agent Framework -->
</ItemGroup>
```

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\..\src\Braintrust.Sdk\Braintrust.Sdk.csproj" />
<ProjectReference Include="..\..\src\Braintrust.Sdk.Extensions.AI\Braintrust.Sdk.Extensions.AI.csproj" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.AI.OpenAI" Version="10.4.1" />
<PackageReference Include="OpenAI" Version="2.9.1" />
</ItemGroup>

</Project>
72 changes: 72 additions & 0 deletions examples/ExtensionsAIInstrumentation/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
using Braintrust.Sdk;
using Braintrust.Sdk.Extensions.AI;
using Microsoft.Extensions.AI;

namespace Braintrust.Sdk.Examples.ExtensionsAIInstrumentation;

/// <summary>
/// Example demonstrating Braintrust instrumentation via Microsoft.Extensions.AI IChatClient.
/// Works with any provider: OpenAI, Azure OpenAI, Ollama, etc.
///
/// Spans emitted include both braintrust.* (for Braintrust dashboard) and gen_ai.*
/// (OTEL GenAI semantic conventions) attributes.
/// </summary>
class Program
{
static async Task Main(string[] args)
{
var openAIApiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY");
if (string.IsNullOrEmpty(openAIApiKey))
{
Console.WriteLine("ERROR: OPENAI_API_KEY environment variable not set. Bailing.");
return;
}

var braintrust = Braintrust.Get();
var activitySource = braintrust.GetActivitySource();

// Create an IChatClient from any provider — here using OpenAI via M.E.AI adapter
var openAIClient = new OpenAI.OpenAIClient(openAIApiKey);
IChatClient chatClient = openAIClient.GetChatClient("gpt-4o-mini").AsIChatClient();

// Add Braintrust tracing at both LLM and function levels
var tracedClient = new ChatClientBuilder(chatClient)
.UseAllBraintrustTracing(activitySource)
.Build();

// Define a tool
var getWeather = AIFunctionFactory.Create(
(string city) => $"The weather in {city} is sunny, 72°F.",
"GetWeather",
"Gets the current weather for a city.");

using (var rootActivity = activitySource.StartActivity("extensions-ai-instrumentation-example"))
{
if (rootActivity != null)
{
Console.WriteLine("~~~ EXTENSIONS.AI INSTRUMENTATION EXAMPLE\n");

// Non-streaming call with tool use
var response = await tracedClient.GetResponseAsync(
[new ChatMessage(ChatRole.User, "What's the weather like in Seattle?")],
new ChatOptions { Tools = [getWeather] });

Console.WriteLine($"Response: {response.Text}");

// Streaming call
Console.Write("\nStreaming: ");
await foreach (var update in tracedClient.GetStreamingResponseAsync(
[new ChatMessage(ChatRole.User, "Tell me a joke.")]))
{
Console.Write(update.Text);
}
Console.WriteLine();

// Print Braintrust link
var url = await braintrust.GetProjectUriAsync()
+ $"/logs?r={rootActivity.TraceId}&s={rootActivity.SpanId}";
Console.WriteLine($"\n View your trace in Braintrust: {url}\n");
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ internal static class BraintrustFunctionMiddleware
if (activity != null)
{
SpanTagHelper.SetSpanType(activity, "function_call");
activity.SetTag("gen_ai.operation.name", "execute_tool");
activity.SetTag("function.name", functionName);
activity.SetTag("gen_ai.tool.name", functionName);
activity.SetTag("function.iteration", context.Iteration);
activity.SetTag("function.call_index", context.FunctionCallIndex);
activity.SetTag("function.total_count", context.FunctionCount);
Expand All @@ -44,8 +46,9 @@ internal static class BraintrustFunctionMiddleware
{
try
{
activity.SetTag("braintrust.input_json",
SpanTagHelper.ToJson(context.Arguments));
var argsJson = SpanTagHelper.ToJson(context.Arguments);
activity.SetTag("braintrust.input_json", argsJson);
activity.SetTag("gen_ai.tool.call.arguments", argsJson);
}
catch
{
Expand Down Expand Up @@ -75,8 +78,9 @@ internal static class BraintrustFunctionMiddleware
{
try
{
activity.SetTag("braintrust.output_json",
SpanTagHelper.ToJson(new { result }));
var resultJson = SpanTagHelper.ToJson(new { result });
activity.SetTag("braintrust.output_json", resultJson);
activity.SetTag("gen_ai.tool.call.result", resultJson);
}
catch
{
Expand Down
24 changes: 17 additions & 7 deletions src/Braintrust.Sdk.Anthropic/InstrumentedMessageService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -166,12 +166,12 @@ public async IAsyncEnumerable<RawMessageStreamEvent> CreateStreaming(
activity.SetTag("braintrust.metrics.time_to_first_token", timeToFirstToken.Value);
}

activity.SetTag(
"braintrust.output_json",
ToJson(new object[]
{
new { role = role ?? "assistant", content = output.ToString() }
}));
var outputJson = ToJson(new object[]
{
new { role = role ?? "assistant", content = output.ToString() }
});
activity.SetTag("braintrust.output_json", outputJson);
activity.SetTag("gen_ai.output.messages", outputJson);
}
}

Expand Down Expand Up @@ -203,6 +203,8 @@ private static void TagActivity(
double? timeToFirstToken = null)
{
activity.SetTag("provider", "anthropic");
activity.SetTag("gen_ai.operation.name", "chat");
activity.SetTag("gen_ai.provider.name", "anthropic");
activity.SetTag("gen_ai.request.model", request.Model.Raw());
activity.SetTag("gen_ai.response.model", response.Model.Raw());

Expand All @@ -221,14 +223,19 @@ private static void TagActivity(
sys.TryPickString(out var sysContent);
inputMessages.Add(new { role = "system", content = sysContent });
}
activity.SetTag("braintrust.input_json", ToJson(inputMessages));
var inputJson = ToJson(inputMessages);
activity.SetTag("braintrust.input_json", inputJson);
activity.SetTag("gen_ai.input.messages", inputJson);

var contentJson = response.ToString();
activity.SetTag("braintrust.output_json", contentJson);
activity.SetTag("gen_ai.output.messages", contentJson);

// Extract token usage metrics
activity.SetTag("braintrust.metrics.prompt_tokens", response.Usage.InputTokens);
activity.SetTag("gen_ai.usage.input_tokens", response.Usage.InputTokens);
activity.SetTag("braintrust.metrics.completion_tokens", response.Usage.OutputTokens);
activity.SetTag("gen_ai.usage.output_tokens", response.Usage.OutputTokens);
activity.SetTag("braintrust.metrics.tokens", response.Usage.InputTokens + response.Usage.OutputTokens);

if (timeToFirstToken is > 0)
Expand Down Expand Up @@ -259,12 +266,15 @@ private static void TagStreamActivity(Activity activity, MessageCreateParams req
{
activity.SetTag("stream", true);
activity.SetTag("provider", "anthropic");
activity.SetTag("gen_ai.operation.name", "chat");
activity.SetTag("gen_ai.provider.name", "anthropic");
activity.SetTag("gen_ai.request.model", request.Model.Raw());

try
{
var messagesJson = ToJson(request.Messages);
activity.SetTag("braintrust.input_json", messagesJson);
activity.SetTag("gen_ai.input.messages", messagesJson);
}
catch
{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<!-- Version is determined dynamically from git state in Directory.Build.targets -->
<PackageReadmeFile>README.md</PackageReadmeFile>
</PropertyGroup>

<ItemGroup>
<InternalsVisibleTo Include="Braintrust.Sdk.Extensions.AI.Tests" />
</ItemGroup>

<ItemGroup>
<None Include="README.md" Pack="true" PackagePath="\" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\Braintrust.Sdk\Braintrust.Sdk.csproj" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.AI" Version="10.4.1" />
</ItemGroup>

</Project>
Loading