From 213c6530ef7700a0f850969c4e7a8981104883c2 Mon Sep 17 00:00:00 2001 From: Matt Perpick Date: Fri, 9 Jan 2026 10:44:22 -0500 Subject: [PATCH 01/10] handle prompts, templating, etc. --- .claude/settings.json | 47 ++ examples/prompt.rb | 95 ++++ lib/braintrust.rb | 1 + lib/braintrust/api/functions.rb | 8 + lib/braintrust/prompt.rb | 172 +++++++ test/braintrust/api/functions_test.rb | 51 ++ test/braintrust/prompt_test.rb | 203 ++++++++ test/fixtures/vcr_cassettes/functions/get.yml | 449 ++++++++++++++++++ test/fixtures/vcr_cassettes/prompt/load.yml | 389 +++++++++++++++ .../vcr_cassettes/prompt/load_not_found.yml | 143 ++++++ 10 files changed, 1558 insertions(+) create mode 100644 .claude/settings.json create mode 100644 examples/prompt.rb create mode 100644 lib/braintrust/prompt.rb create mode 100644 test/braintrust/prompt_test.rb create mode 100644 test/fixtures/vcr_cassettes/functions/get.yml create mode 100644 test/fixtures/vcr_cassettes/prompt/load.yml create mode 100644 test/fixtures/vcr_cassettes/prompt/load_not_found.yml diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..21984fd --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,47 @@ +{ + "permissions": { + "allow": [ + "Bash(git log:*)", + "Bash(git branch:*)", + "Bash(git remote:*)", + "Bash(git fetch:*)", + "Bash(git stash:*)", + "Bash(git diff:*)", + "Bash(git status:*)", + "Bash(git show:*)", + "Bash(gh repo view:*)", + "Bash(gh run list:*)", + "Bash(gh run view:*)", + "Bash(gh pr view:*)", + "Bash(gh pr diff:*)", + "Bash(gh pr list:*)", + "Bash(gh issue view:*)", + "Bash(gh release:*)", + "Bash(gh api:*)", + "Bash(rake -T:*)", + "Bash(bundle show:*)", + "Bash(bundle list:*)", + "Bash(bundle platform:*)", + "Bash(gem search:*)", + "Bash(gem list:*)", + "Bash(tree:*)", + "Bash(find:*)", + "Bash(echo:*)", + "Bash(sort:*)", + "Bash(cat:*)", + "Bash(head:*)", + "Bash(tail:*)", + "Bash(less:*)", + "Bash(wc:*)", + "Bash(ls:*)", + "Bash(pwd:*)", + "Bash(which:*)", + "Bash(type:*)", + "Bash(file:*)", + "Bash(VCR_MODE=new_episodes bundle exec ruby:*)", + "Bash(VCR_MODE=new_episodes bundle exec rake:*)", + "Bash(VCR_MODE=new_episodes rake test:*)" + ], + "deny": [] + } +} diff --git a/examples/prompt.rb b/examples/prompt.rb new file mode 100644 index 0000000..32a51e3 --- /dev/null +++ b/examples/prompt.rb @@ -0,0 +1,95 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Example: Loading and using prompts from Braintrust +# +# This example demonstrates how to: +# 1. Create a prompt (function) on the Braintrust server +# 2. Load it using Prompt.load +# 3. Build the prompt with variable substitution +# 4. Use the built prompt with an LLM client +# +# Benefits of loading prompts: +# - Centralized prompt management in Braintrust UI +# - Version control and A/B testing for prompts +# - No code deployment needed for prompt changes +# - Works with any LLM client (OpenAI, Anthropic, etc.) + +require "bundler/setup" +require "braintrust" + +# Initialize Braintrust +Braintrust.init + +project_name = "ruby-sdk-examples" +prompt_slug = "greeting-prompt-#{Time.now.to_i}" + +# First, create a prompt on the server +# In practice, you would create prompts via the Braintrust UI +puts "Creating prompt..." + +api = Braintrust::API.new +api.functions.create( + project_name: project_name, + slug: prompt_slug, + function_data: {type: "prompt"}, + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "system", + content: "You are a friendly assistant who speaks {{language}}." + }, + { + role: "user", + content: "Say hello to {{name}} and wish them a great {{time_of_day}}!" + } + ] + }, + options: { + model: "gpt-4o-mini", + params: {temperature: 0.7, max_tokens: 100} + } + } +) +puts "Created prompt: #{prompt_slug}" + +# Load the prompt using Prompt.load +puts "\nLoading prompt..." +prompt = Braintrust::Prompt.load(project: project_name, slug: prompt_slug) + +puts " ID: #{prompt.id}" +puts " Name: #{prompt.name}" +puts " Model: #{prompt.model}" +puts " Messages: #{prompt.messages.length}" + +# Build the prompt with variable substitution +puts "\nBuilding prompt with variables..." +params = prompt.build( + name: "Alice", + language: "Spanish", + time_of_day: "morning" +) + +puts " Model: #{params[:model]}" +puts " Temperature: #{params[:temperature]}" +puts " Max tokens: #{params[:max_tokens]}" +puts " Messages:" +params[:messages].each do |msg| + puts " [#{msg[:role]}] #{msg[:content]}" +end + +# The params hash is ready to pass to any LLM client: +# +# With OpenAI: +# client.chat.completions.create(**params) +# +# With Anthropic: +# client.messages.create(**params) + +puts "\nPrompt is ready to use with any LLM client!" + +# Clean up - delete the test prompt +api.functions.delete(id: prompt.id) +puts "Cleaned up test prompt." diff --git a/lib/braintrust.rb b/lib/braintrust.rb index ba023fc..7a61327 100644 --- a/lib/braintrust.rb +++ b/lib/braintrust.rb @@ -5,6 +5,7 @@ require_relative "braintrust/state" require_relative "braintrust/trace" require_relative "braintrust/api" +require_relative "braintrust/prompt" require_relative "braintrust/internal/experiments" require_relative "braintrust/internal/env" require_relative "braintrust/eval" diff --git a/lib/braintrust/api/functions.rb b/lib/braintrust/api/functions.rb index ccfea31..66d82b0 100644 --- a/lib/braintrust/api/functions.rb +++ b/lib/braintrust/api/functions.rb @@ -85,6 +85,14 @@ def invoke(id:, input:) http_post_json("/v1/function/#{id}/invoke", payload) end + # Get a function by ID (includes full prompt_data) + # GET /v1/function/{id} + # @param id [String] Function UUID + # @return [Hash] Full function data including prompt_data + def get(id:) + http_get("/v1/function/#{id}") + end + # Delete a function by ID # DELETE /v1/function/{id} # @param id [String] Function UUID diff --git a/lib/braintrust/prompt.rb b/lib/braintrust/prompt.rb new file mode 100644 index 0000000..d417e95 --- /dev/null +++ b/lib/braintrust/prompt.rb @@ -0,0 +1,172 @@ +# frozen_string_literal: true + +module Braintrust + # Prompt class for loading and building prompts from Braintrust + # + # @example Load and use a prompt + # prompt = Braintrust::Prompt.load(project: "my-project", slug: "summarizer") + # params = prompt.build(text: "Article to summarize...") + # client.messages.create(**params) + class Prompt + attr_reader :id, :name, :slug, :project_id + + # Load a prompt from Braintrust + # + # @param project [String] Project name + # @param slug [String] Prompt slug + # @param version [String, nil] Specific version (default: latest) + # @param defaults [Hash] Default variable values for build() + # @param state [State, nil] Braintrust state (default: global) + # @return [Prompt] + def self.load(project:, slug:, version: nil, defaults: {}, state: nil) + state ||= Braintrust.current_state + raise Error, "No state available - call Braintrust.init first" unless state + + api = API.new(state: state) + + # Find the function by project + slug + result = api.functions.list(project_name: project, slug: slug) + function = result.dig("objects")&.first + raise Error, "Prompt '#{slug}' not found in project '#{project}'" unless function + + # Fetch full function data including prompt_data + full_data = api.functions.get(id: function["id"]) + + new(full_data, defaults: defaults) + end + + # Initialize a Prompt from function data + # + # @param data [Hash] Function data from API + # @param defaults [Hash] Default variable values for build() + def initialize(data, defaults: {}) + @data = data + @defaults = stringify_keys(defaults) + + @id = data["id"] + @name = data["name"] + @slug = data["slug"] + @project_id = data["project_id"] + end + + # Get the raw prompt definition + # @return [Hash, nil] + def prompt + @data.dig("prompt_data", "prompt") + end + + # Get the prompt messages + # @return [Array] + def messages + prompt&.dig("messages") || [] + end + + # Get the model name + # @return [String, nil] + def model + @data.dig("prompt_data", "options", "model") + end + + # Get model options + # @return [Hash] + def options + @data.dig("prompt_data", "options") || {} + end + + # Build the prompt with variable substitution + # + # Returns a hash ready to pass to an LLM client: + # {model: "...", messages: [...], temperature: ..., ...} + # + # @param variables [Hash] Variables to substitute (e.g., {name: "Alice"}) + # @param strict [Boolean] Raise error on missing variables (default: false) + # @return [Hash] Built prompt ready for LLM client + # + # @example With keyword arguments + # prompt.build(name: "Alice", task: "coding") + # + # @example With explicit hash + # prompt.build({name: "Alice"}, strict: true) + def build(variables = nil, strict: false, **kwargs) + # Support both explicit hash and keyword arguments + variables_hash = variables.is_a?(Hash) ? variables : {} + vars = @defaults.merge(stringify_keys(variables_hash)).merge(stringify_keys(kwargs)) + + # Substitute variables in messages + built_messages = messages.map do |msg| + { + role: msg["role"].to_sym, + content: substitute_variables(msg["content"], vars, strict: strict) + } + end + + # Build result with model and messages + result = { + model: model, + messages: built_messages + } + + # Add params (temperature, max_tokens, etc.) to top level + params = options.dig("params") + if params.is_a?(Hash) + params.each do |key, value| + result[key.to_sym] = value + end + end + + result + end + + private + + # Substitute {{variable}} placeholders with values + def substitute_variables(text, variables, strict:) + return text unless text.is_a?(String) + + # Find all {{variable}} patterns + missing = [] + + result = text.gsub(/\{\{([^}]+)\}\}/) do |match| + var_path = ::Regexp.last_match(1).strip + value = resolve_variable(var_path, variables) + + if value.nil? + missing << var_path + match # Keep original placeholder + else + value.to_s + end + end + + if strict && missing.any? + raise Error, "Missing required variables: #{missing.join(", ")}" + end + + result + end + + # Resolve a variable path like "user.name" from variables hash + def resolve_variable(path, variables) + parts = path.split(".") + value = variables + + parts.each do |part| + return nil unless value.is_a?(Hash) + # Try both string and symbol keys + value = value[part] || value[part.to_sym] + return nil if value.nil? + end + + value + end + + # Convert hash keys to strings (handles both symbol and string keys) + def stringify_keys(hash) + return {} unless hash.is_a?(Hash) + + hash.transform_keys(&:to_s).transform_values do |v| + v.is_a?(Hash) ? stringify_keys(v) : v + end + end + end +end diff --git a/test/braintrust/api/functions_test.rb b/test/braintrust/api/functions_test.rb index 753c1d2..bdb6e5b 100644 --- a/test/braintrust/api/functions_test.rb +++ b/test/braintrust/api/functions_test.rb @@ -378,4 +378,55 @@ def test_helper_validates_prompt_data_has_prompt_key end end end + + def test_functions_get_by_id + VCR.use_cassette("functions/get") do + api = get_test_api + # This test verifies that we can get the full function data by ID, + # including prompt_data which is needed for Prompt.load() + function_slug = "test-ruby-sdk-get-func" + + # Create a function with prompt_data + create_response = api.functions.create( + project_name: @project_name, + slug: function_slug, + function_data: {type: "prompt"}, + prompt_data: { + prompt: { + type: "chat", + messages: [ + {role: "system", content: "You are a helpful assistant."}, + {role: "user", content: "Hello {{name}}"} + ] + }, + options: { + model: "gpt-4o-mini", + params: {temperature: 0.7} + } + } + ) + function_id = create_response["id"] + + # Get the full function data + result = api.functions.get(id: function_id) + + assert_instance_of Hash, result + assert_equal function_id, result["id"] + assert_equal function_slug, result["slug"] + + # Verify prompt_data is included + assert result.key?("prompt_data") + prompt_data = result["prompt_data"] + assert prompt_data.key?("prompt") + assert prompt_data["prompt"].key?("messages") + assert_equal 2, prompt_data["prompt"]["messages"].length + + # Verify options are included + assert prompt_data.key?("options") + assert_equal "gpt-4o-mini", prompt_data["options"]["model"] + + # Clean up + api.functions.delete(id: function_id) + end + end end diff --git a/test/braintrust/prompt_test.rb b/test/braintrust/prompt_test.rb new file mode 100644 index 0000000..14ed7d1 --- /dev/null +++ b/test/braintrust/prompt_test.rb @@ -0,0 +1,203 @@ +# frozen_string_literal: true + +require "test_helper" + +class Braintrust::PromptTest < Minitest::Test + def setup + # Sample function data as returned by API + @function_data = { + "id" => "func-123", + "name" => "Test Prompt", + "slug" => "test-prompt", + "project_id" => "proj-456", + "prompt_data" => { + "prompt" => { + "type" => "chat", + "messages" => [ + {"role" => "system", "content" => "You are a helpful assistant."}, + {"role" => "user", "content" => "Hello {{name}}, please help with {{task}}."} + ] + }, + "options" => { + "model" => "claude-3-5-sonnet", + "params" => {"temperature" => 0.7, "max_tokens" => 1000} + } + } + } + end + + def test_prompt_initialization + prompt = Braintrust::Prompt.new(@function_data) + + assert_equal "func-123", prompt.id + assert_equal "Test Prompt", prompt.name + assert_equal "test-prompt", prompt.slug + assert_equal "proj-456", prompt.project_id + end + + def test_prompt_messages + prompt = Braintrust::Prompt.new(@function_data) + messages = prompt.messages + + assert_equal 2, messages.length + assert_equal "system", messages[0]["role"] + assert_equal "You are a helpful assistant.", messages[0]["content"] + assert_equal "user", messages[1]["role"] + end + + def test_prompt_model + prompt = Braintrust::Prompt.new(@function_data) + + assert_equal "claude-3-5-sonnet", prompt.model + end + + def test_prompt_options + prompt = Braintrust::Prompt.new(@function_data) + options = prompt.options + + assert_equal "claude-3-5-sonnet", options["model"] + assert_equal 0.7, options["params"]["temperature"] + assert_equal 1000, options["params"]["max_tokens"] + end + + def test_prompt_raw_returns_prompt_definition + prompt = Braintrust::Prompt.new(@function_data) + raw = prompt.prompt + + assert_equal "chat", raw["type"] + assert raw.key?("messages") + end + + def test_build_substitutes_variables + prompt = Braintrust::Prompt.new(@function_data) + result = prompt.build(name: "Alice", task: "coding") + + assert_equal "claude-3-5-sonnet", result[:model] + assert_equal 2, result[:messages].length + assert_equal "You are a helpful assistant.", result[:messages][0][:content] + assert_equal "Hello Alice, please help with coding.", result[:messages][1][:content] + end + + def test_build_with_string_keys + prompt = Braintrust::Prompt.new(@function_data) + result = prompt.build("name" => "Bob", "task" => "writing") + + assert_equal "Hello Bob, please help with writing.", result[:messages][1][:content] + end + + def test_build_includes_params + prompt = Braintrust::Prompt.new(@function_data) + result = prompt.build(name: "Alice", task: "coding") + + assert_equal 0.7, result[:temperature] + assert_equal 1000, result[:max_tokens] + end + + def test_build_with_defaults + prompt = Braintrust::Prompt.new(@function_data, defaults: {name: "Default User"}) + result = prompt.build(task: "testing") + + assert_equal "Hello Default User, please help with testing.", result[:messages][1][:content] + end + + def test_build_overrides_defaults + prompt = Braintrust::Prompt.new(@function_data, defaults: {name: "Default User"}) + result = prompt.build(name: "Override User", task: "testing") + + assert_equal "Hello Override User, please help with testing.", result[:messages][1][:content] + end + + def test_build_strict_raises_on_missing_variable + prompt = Braintrust::Prompt.new(@function_data) + + error = assert_raises(Braintrust::Error) do + prompt.build({name: "Alice"}, strict: true) + end + + assert_match(/missing.*task/i, error.message) + end + + def test_build_non_strict_leaves_missing_variables + prompt = Braintrust::Prompt.new(@function_data) + result = prompt.build(name: "Alice") + + assert_equal "Hello Alice, please help with {{task}}.", result[:messages][1][:content] + end + + def test_build_handles_nested_variables + data = @function_data.dup + data["prompt_data"]["prompt"]["messages"] = [ + {"role" => "user", "content" => "User: {{user.name}}, Email: {{user.email}}"} + ] + prompt = Braintrust::Prompt.new(data) + + result = prompt.build(user: {name: "Alice", email: "alice@example.com"}) + + assert_equal "User: Alice, Email: alice@example.com", result[:messages][0][:content] + end +end + +class Braintrust::PromptLoadTest < Minitest::Test + def setup + flunk "BRAINTRUST_API_KEY not set" unless ENV["BRAINTRUST_API_KEY"] + @project_name = "ruby-sdk-test" + end + + def test_prompt_load + VCR.use_cassette("prompt/load") do + Braintrust.init(blocking_login: true) + + # Create a prompt first + api = Braintrust::API.new + slug = "test-prompt-load" + + api.functions.create( + project_name: @project_name, + slug: slug, + function_data: {type: "prompt"}, + prompt_data: { + prompt: { + type: "chat", + messages: [ + {role: "user", content: "Say hello to {{name}}"} + ] + }, + options: { + model: "gpt-4o-mini" + } + } + ) + + # Load the prompt using Prompt.load + prompt = Braintrust::Prompt.load(project: @project_name, slug: slug) + + assert_instance_of Braintrust::Prompt, prompt + assert_equal slug, prompt.slug + assert_equal "gpt-4o-mini", prompt.model + assert_equal 1, prompt.messages.length + + # Test build + result = prompt.build(name: "World") + assert_equal "Say hello to World", result[:messages][0][:content] + + # Clean up + api.functions.delete(id: prompt.id) + ensure + OpenTelemetry.tracer_provider.shutdown + end + end + + def test_prompt_load_not_found + VCR.use_cassette("prompt/load_not_found") do + Braintrust.init(blocking_login: true) + + error = assert_raises(Braintrust::Error) do + Braintrust::Prompt.load(project: @project_name, slug: "nonexistent-prompt-xyz") + end + + assert_match(/not found/i, error.message) + ensure + OpenTelemetry.tracer_provider.shutdown + end + end +end diff --git a/test/fixtures/vcr_cassettes/functions/get.yml b/test/fixtures/vcr_cassettes/functions/get.yml new file mode 100644 index 0000000..908caac --- /dev/null +++ b/test/fixtures/vcr_cassettes/functions/get.yml @@ -0,0 +1,449 @@ +--- +http_interactions: +- request: + method: get + uri: https://api.braintrust.dev/v1/function/eaa1d7fa-ddfc-487e-bdf1-8c83c9e5e216 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '650' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:22:45 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 999f045e-52bf-431a-ab23-4231e84c599b + X-Bt-Internal-Trace-Id: + - 69611d45000000001d09e277b8e05ffb + X-Amz-Apigw-Id: + - W7GC7HqrIAMEBEQ= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"28a-o+EOs8vizZBvL0+kfKefNOgxPc0" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611d45-406960f37e85c137106930c0;Parent=0943949cf2b37855;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 f8debc28b6c73eb3dc7540e2ac2f0e18.cloudfront.net (CloudFront), 1.1 d9b04a822e1c215374729ec159356140.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - UeZO45ONY1S230A3ccsinel_eSg2umCVj5z3YObzFX1Y_j2Q3IXy6w== + body: + encoding: ASCII-8BIT + string: '{"id":"eaa1d7fa-ddfc-487e-bdf1-8c83c9e5e216","_xact_id":"1000196458014160880","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-ruby-sdk-get-func","slug":"test-ruby-sdk-get-func","description":null,"created":"2026-01-09T15:06:49.196Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + are a helpful assistant."},{"role":"user","content":"Hello {{name}}"}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0.7}}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}' + recorded_at: Fri, 09 Jan 2026 15:22:45 GMT +- request: + method: delete + uri: https://api.braintrust.dev/v1/function/eaa1d7fa-ddfc-487e-bdf1-8c83c9e5e216 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '282' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:22:47 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - b46eee9c-fcb5-4035-897c-fae539611a0e + X-Bt-Internal-Trace-Id: + - 69611d47000000005cc7572f079de445 + X-Amz-Apigw-Id: + - W7GDKGoaoAMEg5g= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"11a-P+3A9L3licMg9JgEGiq9t1aKens" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611d47-4a348ed101c9c9fc6fdb1b86;Parent=14e14957b2cba183;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 b601959712c1f21193a489b5759f70ba.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - VsS0_f9QROIT-ckZpbiAmMpiT6skqE6pHr-hGzREkFyqEXIihNwgNA== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-get-func","id":"eaa1d7fa-ddfc-487e-bdf1-8c83c9e5e216","created":"2026-01-09T15:22:47.106Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196458076954719"}' + recorded_at: Fri, 09 Jan 2026 15:22:47 GMT +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MzQ5OGZkZjYtMmJlOC00NDY2LWE1MzAtODY5N2ViZjI5YzFl'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 09 Jan 2026 15:26:29 GMT + Etag: + - '"ubzjf1iqqj75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - MzQ5OGZkZjYtMmJlOC00NDY2LWE1MzAtODY5N2ViZjI5YzFl + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - iad1::iad1::vhxvp-1767972389112-bcc3207d4501 + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Fri, 09 Jan 2026 15:26:29 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/project?project_name=ruby-sdk-test + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '269' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:26:29 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - df8c7dca-c9f8-4554-bc08-c4003c5faaaa + X-Bt-Internal-Trace-Id: + - 69611e25000000002d2feed003d7844a + X-Amz-Apigw-Id: + - W7Gl4FNZIAMET2Q= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"10d-jHt6t+s3DyzhLf+d20I2XKvIjGQ" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611e25-0e71a6817e098cd12d46df1e;Parent=55ae8824648bf68a;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 f25b89e7ef738cb8bb7e28e041d8fe54.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - Z8Zuf3NfTPdgGMykdiyvlEx-Hfpr-8BgDvslAhNlF-FBzdse37eiWQ== + body: + encoding: ASCII-8BIT + string: '{"objects":[{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","description":null,"created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}]}' + recorded_at: Fri, 09 Jan 2026 15:26:29 GMT +- request: + method: post + uri: https://api.braintrust.dev/v1/function + body: + encoding: UTF-8 + string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-get-func","name":"test-ruby-sdk-get-func","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + are a helpful assistant."},{"role":"user","content":"Hello {{name}}"}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0.7}}}}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '545' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:26:29 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 601a8e03-eb46-49c4-8d2a-facdecc0274a + X-Bt-Internal-Trace-Id: + - 69611e25000000004e9aa6fbc35c2222 + X-Amz-Apigw-Id: + - W7Gl8EE0IAMEBuw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"221-MYmA2S4f3IZPIdPfajiNHs1EESA" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611e25-605279d552cb95a6336cea62;Parent=3d0b812115b7dd19;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 c4d0da6268789cfda9bb5da1f3f8fc58.cloudfront.net (CloudFront), 1.1 537c1727cc67e6d2567bb61ae0478182.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 5P3DoYCNcIvlRYIrkJ6CQY-dCflpP0eCy6XrAmn2kJObAN6B1_kgsA== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-get-func","name":"test-ruby-sdk-get-func","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + are a helpful assistant."},{"role":"user","content":"Hello {{name}}"}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0.7}}},"id":"9c763edd-0474-4034-8869-8bbe307fcd32","created":"2026-01-09T15:26:29.742Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196458091495183"}' + recorded_at: Fri, 09 Jan 2026 15:26:29 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/function/9c763edd-0474-4034-8869-8bbe307fcd32 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '650' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:26:30 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 9a293bcc-24ab-4ec0-9b3f-842c0fce7b76 + X-Bt-Internal-Trace-Id: + - 69611e250000000002effecf6f118ea1 + X-Amz-Apigw-Id: + - W7Gl_FW4IAMEI8w= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"28a-dA3pKxLHq2B0teWLcR3HocUdXLs" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611e25-2ebc8ae9137051f60738d543;Parent=66f232e4b293ec11;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 ab734ad5d81cc9d470b6176a05dd968e.cloudfront.net (CloudFront), 1.1 8ca36406fe3aa11c1641e5bc917c8a74.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - vOXNwjsPTHAO-Am1MlqnlLjUiJfihdYMdlRvPf-vQW1B3CP7ogmIAw== + body: + encoding: ASCII-8BIT + string: '{"id":"9c763edd-0474-4034-8869-8bbe307fcd32","_xact_id":"1000196458091495183","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-ruby-sdk-get-func","slug":"test-ruby-sdk-get-func","description":null,"created":"2026-01-09T15:26:29.742Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + are a helpful assistant."},{"role":"user","content":"Hello {{name}}"}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0.7}}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}' + recorded_at: Fri, 09 Jan 2026 15:26:30 GMT +- request: + method: delete + uri: https://api.braintrust.dev/v1/function/9c763edd-0474-4034-8869-8bbe307fcd32 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '282' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:26:30 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - d1b4a95f-c311-4b54-8332-a8d893a10e09 + X-Bt-Internal-Trace-Id: + - 69611e260000000011fd6ec6f9aad01f + X-Amz-Apigw-Id: + - W7GmDGdooAMERvQ= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"11a-G6BnvsesfXj6YiI3nDotOemHZaM" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611e26-6993aa0d4516e67113188a3e;Parent=7678979c42d17017;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 b5fe18267507cb61755963d8928a60f4.cloudfront.net (CloudFront), 1.1 b601959712c1f21193a489b5759f70ba.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 8gF7-nfICr1kX_ArRuSfyuBcWIjLCSubLVjOW_gAi3mvyB_fNxsaxw== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-get-func","id":"9c763edd-0474-4034-8869-8bbe307fcd32","created":"2026-01-09T15:26:30.444Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196458091561660"}' + recorded_at: Fri, 09 Jan 2026 15:26:30 GMT +recorded_with: VCR 6.4.0 diff --git a/test/fixtures/vcr_cassettes/prompt/load.yml b/test/fixtures/vcr_cassettes/prompt/load.yml new file mode 100644 index 0000000..9df95d7 --- /dev/null +++ b/test/fixtures/vcr_cassettes/prompt/load.yml @@ -0,0 +1,389 @@ +--- +http_interactions: +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MGQzNDRiYjctMzcyZC00ODEzLTg0MmYtYjYwNmFiZWY0Nzc5'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 09 Jan 2026 15:31:57 GMT + Etag: + - '"ubzjf1iqqj75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - MGQzNDRiYjctMzcyZC00ODEzLTg0MmYtYjYwNmFiZWY0Nzc5 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - iad1::iad1::478kl-1767972717018-8f1c4449dd71 + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Fri, 09 Jan 2026 15:31:57 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/project?project_name=ruby-sdk-test + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '269' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:57 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 6308ae43-6aef-4e2a-b677-6eeb03896599 + X-Bt-Internal-Trace-Id: + - 69611f6d000000007e25f8ac6b23187b + X-Amz-Apigw-Id: + - W7HZIGr1IAMEjxA= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"10d-jHt6t+s3DyzhLf+d20I2XKvIjGQ" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6d-6e2659e44897bfc601214fab;Parent=168da3c03f62beb4;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 ad22d4e4410fd07809425488bf6e79be.cloudfront.net (CloudFront), 1.1 f391dfb0806f29cccc5f1df3e1ae836e.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - dEzh8blLXvYXHkMRSBV5zi5DiwpekBQFkgXMWtmtiirn4jCsm4dU8w== + body: + encoding: ASCII-8BIT + string: '{"objects":[{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","description":null,"created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}]}' + recorded_at: Fri, 09 Jan 2026 15:31:57 GMT +- request: + method: post + uri: https://api.braintrust.dev/v1/function + body: + encoding: UTF-8 + string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-prompt-load","name":"test-prompt-load","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Say + hello to {{name}}"}]},"options":{"model":"gpt-4o-mini"}}}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '452' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:57 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 411104c9-b444-450f-b49c-d7cd030b9b18 + X-Bt-Internal-Trace-Id: + - 69611f6d000000007281a26ff26ab68f + X-Amz-Apigw-Id: + - W7HZKEbxIAMEDJQ= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1c4-HTfX1Q46VKOG3lasMWxS8J85LCQ" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6d-27b6279359f4794e35dcdbc5;Parent=5158f835fa228c4a;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 6ea9421ec132e3640100792ef9535494.cloudfront.net (CloudFront), 1.1 d3041c3025b9205db460853b5b9626bc.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - hL_HuFng0i2Jeib3N83UPHx5g9XfA0nnGHSST1ARKxpzagyH3dd-2g== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-prompt-load","name":"test-prompt-load","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Say + hello to {{name}}"}]},"options":{"model":"gpt-4o-mini"}},"id":"2ee042ed-2971-47f4-a73c-7abe38ab1a38","created":"2026-01-09T15:31:57.524Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196458112984917"}' + recorded_at: Fri, 09 Jan 2026 15:31:57 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/function?project_name=ruby-sdk-test&slug=test-prompt-load + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '571' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:58 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - af75d70c-ba8f-4244-a61e-fb6e643abd74 + X-Bt-Internal-Trace-Id: + - 69611f6d0000000079da7c9d7c78e29e + X-Amz-Apigw-Id: + - W7HZNF8ooAMErzg= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"23b-zKRi5DEcuekPd/TsGth74hG9tXw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6d-17dd36184ab7c9202e705efa;Parent=4103abe6bc9216e0;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 ef73a156d5c211fdbb7e4231f2a0edca.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 78ntxSmfza39FAmaCa-exUtRnNXyNWRnHxcQ77_UZoLQ1D-gcWcmHg== + body: + encoding: ASCII-8BIT + string: '{"objects":[{"id":"2ee042ed-2971-47f4-a73c-7abe38ab1a38","_xact_id":"1000196458112984917","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-prompt-load","slug":"test-prompt-load","description":null,"created":"2026-01-09T15:31:57.524Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Say + hello to {{name}}"}]},"options":{"model":"gpt-4o-mini"}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}]}' + recorded_at: Fri, 09 Jan 2026 15:31:58 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/function/2ee042ed-2971-47f4-a73c-7abe38ab1a38 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '557' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:58 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 9e3c6ab3-0160-4a1e-94a9-35ab9f19913e + X-Bt-Internal-Trace-Id: + - 69611f6e0000000071b8a71148470d4f + X-Amz-Apigw-Id: + - W7HZTHd7oAMESTw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"22d-ma3D4t+NO2VLj1JEXruVnj40R7s" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6e-71c865c4585fcb7f648bd594;Parent=407f3261d889f0cb;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 f9aa0e4086fcbefc20f307d96a8e3b44.cloudfront.net (CloudFront), 1.1 b601959712c1f21193a489b5759f70ba.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - ccO14e2OKjR_YWWVAAiYlvQ148Mh72XPFQUJP7-mfkXI8h4tmQjX1g== + body: + encoding: ASCII-8BIT + string: '{"id":"2ee042ed-2971-47f4-a73c-7abe38ab1a38","_xact_id":"1000196458112984917","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-prompt-load","slug":"test-prompt-load","description":null,"created":"2026-01-09T15:31:57.524Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Say + hello to {{name}}"}]},"options":{"model":"gpt-4o-mini"}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}' + recorded_at: Fri, 09 Jan 2026 15:31:58 GMT +- request: + method: delete + uri: https://api.braintrust.dev/v1/function/2ee042ed-2971-47f4-a73c-7abe38ab1a38 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '276' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:59 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 2fd59f3d-1ab3-4711-a36d-cc8f4a571f38 + X-Bt-Internal-Trace-Id: + - 69611f6e000000006407a3df038879bd + X-Amz-Apigw-Id: + - W7HZYFE2oAMEF7g= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"114-DQ3SOr2UfgKZ6FbXE8ta7C0iQ+0" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6e-4980fe2e0131dc962a5a453b;Parent=21b6b931be060230;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 38bc9c97daf30f968ccac44ef89e14e0.cloudfront.net (CloudFront), 1.1 92672fff57a11d8cf4f64313a69242d0.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - t19MgdmAEgAvifEHWTWB_Eg5GrWxHuhgmBzUCXkGWFh0UPmXSQsLKA== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-prompt-load","id":"2ee042ed-2971-47f4-a73c-7abe38ab1a38","created":"2026-01-09T15:31:58.990Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196458113118216"}' + recorded_at: Fri, 09 Jan 2026 15:31:59 GMT +recorded_with: VCR 6.4.0 diff --git a/test/fixtures/vcr_cassettes/prompt/load_not_found.yml b/test/fixtures/vcr_cassettes/prompt/load_not_found.yml new file mode 100644 index 0000000..aeafd82 --- /dev/null +++ b/test/fixtures/vcr_cassettes/prompt/load_not_found.yml @@ -0,0 +1,143 @@ +--- +http_interactions: +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZDE5MmFhODgtNjU5Ny00OWVmLTgzZjAtNDRmMDJlYTY5MmQ1'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 09 Jan 2026 15:32:03 GMT + Etag: + - '"ubzjf1iqqj75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - ZDE5MmFhODgtNjU5Ny00OWVmLTgzZjAtNDRmMDJlYTY5MmQ1 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - iad1::iad1::c7rlk-1767972723551-c064f07e7248 + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Fri, 09 Jan 2026 15:32:03 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/function?project_name=ruby-sdk-test&slug=nonexistent-prompt-xyz + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '14' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:32:03 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - e525a4b0-3de5-43ce-9b4f-fedef9b6d867 + X-Bt-Internal-Trace-Id: + - 69611f73000000005ccf167cee9e9974 + X-Amz-Apigw-Id: + - W7HaJFBAIAMEF8w= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"e-xZKibKAiOxxBbzTm2byfFNRkvtA" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f73-2656d761732ae2725b2c54f8;Parent=0f5f3e756009b8f2;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 ab734ad5d81cc9d470b6176a05dd968e.cloudfront.net (CloudFront), 1.1 baec235d174153a8f2e92ea724643824.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - CcRV9GjE2YYavLYeGWNDevFa9mY2YldGSV-TUg4E5QrLLfsnxL3cJQ== + body: + encoding: ASCII-8BIT + string: '{"objects":[]}' + recorded_at: Fri, 09 Jan 2026 15:32:03 GMT +recorded_with: VCR 6.4.0 From 5321ce07e7de23a8d1729f4cc80ccbb813359811 Mon Sep 17 00:00:00 2001 From: Matt Perpick Date: Fri, 9 Jan 2026 11:27:22 -0500 Subject: [PATCH 02/10] use mustache templates --- .claude/settings.json | 1 + Gemfile.lock | 2 ++ braintrust.gemspec | 1 + examples/prompt.rb | 41 ++++++++++++++++++--------------- lib/braintrust/prompt.rb | 42 +++++++++++++++++----------------- test/braintrust/prompt_test.rb | 5 ++-- 6 files changed, 51 insertions(+), 41 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index 21984fd..bba6f84 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -25,6 +25,7 @@ "Bash(gem search:*)", "Bash(gem list:*)", "Bash(tree:*)", + "Bash(grep:*)", "Bash(find:*)", "Bash(echo:*)", "Bash(sort:*)", diff --git a/Gemfile.lock b/Gemfile.lock index 533cb68..36cb4fe 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,6 +2,7 @@ PATH remote: . specs: braintrust (0.1.1) + mustache (~> 1.0) openssl (~> 3.3.1) opentelemetry-exporter-otlp (~> 0.28) opentelemetry-sdk (~> 1.3) @@ -48,6 +49,7 @@ GEM minitest (>= 5.0) ruby-progressbar minitest-stub-const (0.6) + mustache (1.1.1) openssl (3.3.2) opentelemetry-api (1.7.0) opentelemetry-common (0.23.0) diff --git a/braintrust.gemspec b/braintrust.gemspec index ba9f5c3..4bc2c75 100644 --- a/braintrust.gemspec +++ b/braintrust.gemspec @@ -32,6 +32,7 @@ Gem::Specification.new do |spec| # Runtime dependencies spec.add_runtime_dependency "opentelemetry-sdk", "~> 1.3" spec.add_runtime_dependency "opentelemetry-exporter-otlp", "~> 0.28" + spec.add_runtime_dependency "mustache", "~> 1.0" # OpenSSL 3.3.1+ fixes macOS CRL (Certificate Revocation List) verification issues # that occur with OpenSSL 3.6 + Ruby (certificate verify failed: unable to get certificate CRL). diff --git a/examples/prompt.rb b/examples/prompt.rb index 32a51e3..4c0e837 100644 --- a/examples/prompt.rb +++ b/examples/prompt.rb @@ -1,26 +1,31 @@ #!/usr/bin/env ruby # frozen_string_literal: true -# Example: Loading and using prompts from Braintrust +# Example: Loading and executing prompts from Braintrust # # This example demonstrates how to: # 1. Create a prompt (function) on the Braintrust server # 2. Load it using Prompt.load -# 3. Build the prompt with variable substitution -# 4. Use the built prompt with an LLM client +# 3. Build the prompt with Mustache variable substitution +# 4. Execute the prompt with OpenAI and get a response # # Benefits of loading prompts: # - Centralized prompt management in Braintrust UI # - Version control and A/B testing for prompts # - No code deployment needed for prompt changes # - Works with any LLM client (OpenAI, Anthropic, etc.) +# - Uses standard Mustache templating ({{variable}}, {{object.property}}) require "bundler/setup" require "braintrust" +require "openai" -# Initialize Braintrust +# Initialize Braintrust with tracing Braintrust.init +# Wrap OpenAI client for tracing +openai = Braintrust::Trace::OpenAI.wrap(OpenAI::Client.new) + project_name = "ruby-sdk-examples" prompt_slug = "greeting-prompt-#{Time.now.to_i}" @@ -39,7 +44,7 @@ messages: [ { role: "system", - content: "You are a friendly assistant who speaks {{language}}." + content: "You are a friendly assistant. Respond in {{language}}. Keep responses brief (1-2 sentences)." }, { role: "user", @@ -60,11 +65,10 @@ prompt = Braintrust::Prompt.load(project: project_name, slug: prompt_slug) puts " ID: #{prompt.id}" -puts " Name: #{prompt.name}" +puts " Slug: #{prompt.slug}" puts " Model: #{prompt.model}" -puts " Messages: #{prompt.messages.length}" -# Build the prompt with variable substitution +# Build the prompt with Mustache variable substitution puts "\nBuilding prompt with variables..." params = prompt.build( name: "Alice", @@ -74,22 +78,23 @@ puts " Model: #{params[:model]}" puts " Temperature: #{params[:temperature]}" -puts " Max tokens: #{params[:max_tokens]}" puts " Messages:" params[:messages].each do |msg| puts " [#{msg[:role]}] #{msg[:content]}" end -# The params hash is ready to pass to any LLM client: -# -# With OpenAI: -# client.chat.completions.create(**params) -# -# With Anthropic: -# client.messages.create(**params) +# Execute the prompt with OpenAI +puts "\nExecuting prompt with OpenAI..." +response = openai.chat.completions.create(**params) -puts "\nPrompt is ready to use with any LLM client!" +puts "\nResponse:" +content = response.choices.first.message.content +puts " #{content}" # Clean up - delete the test prompt +puts "\nCleaning up..." api.functions.delete(id: prompt.id) -puts "Cleaned up test prompt." +puts "Done!" + +# Flush traces +OpenTelemetry.tracer_provider.shutdown diff --git a/lib/braintrust/prompt.rb b/lib/braintrust/prompt.rb index d417e95..dc162ff 100644 --- a/lib/braintrust/prompt.rb +++ b/lib/braintrust/prompt.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "mustache" + module Braintrust # Prompt class for loading and building prompts from Braintrust # @@ -92,11 +94,11 @@ def build(variables = nil, strict: false, **kwargs) variables_hash = variables.is_a?(Hash) ? variables : {} vars = @defaults.merge(stringify_keys(variables_hash)).merge(stringify_keys(kwargs)) - # Substitute variables in messages + # Render Mustache templates in messages built_messages = messages.map do |msg| { role: msg["role"].to_sym, - content: substitute_variables(msg["content"], vars, strict: strict) + content: render_template(msg["content"], vars, strict: strict) } end @@ -119,33 +121,31 @@ def build(variables = nil, strict: false, **kwargs) private - # Substitute {{variable}} placeholders with values - def substitute_variables(text, variables, strict:) + # Render Mustache template with variables + def render_template(text, variables, strict:) return text unless text.is_a?(String) - # Find all {{variable}} patterns - missing = [] - - result = text.gsub(/\{\{([^}]+)\}\}/) do |match| - var_path = ::Regexp.last_match(1).strip - value = resolve_variable(var_path, variables) - - if value.nil? - missing << var_path - match # Keep original placeholder - else - value.to_s + if strict + # Check for missing variables before rendering + missing = find_missing_variables(text, variables) + if missing.any? + raise Error, "Missing required variables: #{missing.join(", ")}" end end - if strict && missing.any? - raise Error, "Missing required variables: #{missing.join(", ")}" - end + Mustache.render(text, variables) + end - result + # Find variables in template that are not provided + def find_missing_variables(text, variables) + # Extract {{variable}} and {{variable.path}} patterns + # Mustache uses {{name}} syntax + text.scan(/\{\{([^}#^\/!>]+)\}\}/).flatten.map(&:strip).uniq.reject do |var| + resolve_variable(var, variables) + end end - # Resolve a variable path like "user.name" from variables hash + # Check if a variable path exists in the variables hash def resolve_variable(path, variables) parts = path.split(".") value = variables diff --git a/test/braintrust/prompt_test.rb b/test/braintrust/prompt_test.rb index 14ed7d1..247de90 100644 --- a/test/braintrust/prompt_test.rb +++ b/test/braintrust/prompt_test.rb @@ -117,11 +117,12 @@ def test_build_strict_raises_on_missing_variable assert_match(/missing.*task/i, error.message) end - def test_build_non_strict_leaves_missing_variables + def test_build_non_strict_replaces_missing_variables_with_empty + # Mustache standard behavior: missing variables become empty strings prompt = Braintrust::Prompt.new(@function_data) result = prompt.build(name: "Alice") - assert_equal "Hello Alice, please help with {{task}}.", result[:messages][1][:content] + assert_equal "Hello Alice, please help with .", result[:messages][1][:content] end def test_build_handles_nested_variables From a5f4e34a94f275d18c26b0221f4a77cdf1bc5669 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 2 Feb 2026 19:59:22 -0500 Subject: [PATCH 03/10] Changed: Vendored mustache gem --- .standard.yml | 5 + Gemfile.lock | 2 - braintrust.gemspec | 1 - lib/braintrust/prompt.rb | 4 +- lib/braintrust/vendor/mustache.rb | 16 + lib/braintrust/vendor/mustache/context.rb | 180 +++++++++ .../vendor/mustache/context_miss.rb | 22 ++ lib/braintrust/vendor/mustache/enumerable.rb | 14 + lib/braintrust/vendor/mustache/generator.rb | 188 +++++++++ lib/braintrust/vendor/mustache/mustache.rb | 260 +++++++++++++ lib/braintrust/vendor/mustache/parser.rb | 364 ++++++++++++++++++ lib/braintrust/vendor/mustache/settings.rb | 252 ++++++++++++ lib/braintrust/vendor/mustache/template.rb | 138 +++++++ lib/braintrust/vendor/mustache/utils.rb | 42 ++ test/braintrust/prompt_test.rb | 34 ++ 15 files changed, 1517 insertions(+), 5 deletions(-) create mode 100644 .standard.yml create mode 100644 lib/braintrust/vendor/mustache.rb create mode 100644 lib/braintrust/vendor/mustache/context.rb create mode 100644 lib/braintrust/vendor/mustache/context_miss.rb create mode 100644 lib/braintrust/vendor/mustache/enumerable.rb create mode 100644 lib/braintrust/vendor/mustache/generator.rb create mode 100644 lib/braintrust/vendor/mustache/mustache.rb create mode 100644 lib/braintrust/vendor/mustache/parser.rb create mode 100644 lib/braintrust/vendor/mustache/settings.rb create mode 100644 lib/braintrust/vendor/mustache/template.rb create mode 100644 lib/braintrust/vendor/mustache/utils.rb diff --git a/.standard.yml b/.standard.yml new file mode 100644 index 0000000..2bbb355 --- /dev/null +++ b/.standard.yml @@ -0,0 +1,5 @@ +# StandardRB configuration +# https://github.com/standardrb/standard + +ignore: + - "lib/braintrust/vendor/**/*" # Vendored code follows upstream conventions diff --git a/Gemfile.lock b/Gemfile.lock index 36cb4fe..533cb68 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,6 @@ PATH remote: . specs: braintrust (0.1.1) - mustache (~> 1.0) openssl (~> 3.3.1) opentelemetry-exporter-otlp (~> 0.28) opentelemetry-sdk (~> 1.3) @@ -49,7 +48,6 @@ GEM minitest (>= 5.0) ruby-progressbar minitest-stub-const (0.6) - mustache (1.1.1) openssl (3.3.2) opentelemetry-api (1.7.0) opentelemetry-common (0.23.0) diff --git a/braintrust.gemspec b/braintrust.gemspec index 4bc2c75..ba9f5c3 100644 --- a/braintrust.gemspec +++ b/braintrust.gemspec @@ -32,7 +32,6 @@ Gem::Specification.new do |spec| # Runtime dependencies spec.add_runtime_dependency "opentelemetry-sdk", "~> 1.3" spec.add_runtime_dependency "opentelemetry-exporter-otlp", "~> 0.28" - spec.add_runtime_dependency "mustache", "~> 1.0" # OpenSSL 3.3.1+ fixes macOS CRL (Certificate Revocation List) verification issues # that occur with OpenSSL 3.6 + Ruby (certificate verify failed: unable to get certificate CRL). diff --git a/lib/braintrust/prompt.rb b/lib/braintrust/prompt.rb index dc162ff..4aab3e0 100644 --- a/lib/braintrust/prompt.rb +++ b/lib/braintrust/prompt.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "mustache" +require_relative "vendor/mustache" module Braintrust # Prompt class for loading and building prompts from Braintrust @@ -133,7 +133,7 @@ def render_template(text, variables, strict:) end end - Mustache.render(text, variables) + Vendor::Mustache.render(text, variables) end # Find variables in template that are not provided diff --git a/lib/braintrust/vendor/mustache.rb b/lib/braintrust/vendor/mustache.rb new file mode 100644 index 0000000..4267497 --- /dev/null +++ b/lib/braintrust/vendor/mustache.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# Vendored Mustache template engine +# From mustache gem v1.1.1 - https://github.com/mustache/mustache +# License: MIT +# +# Modifications from original: +# - Namespaced under Braintrust::Vendor to avoid conflicts +# - Disabled HTML escaping (LLM prompts don't need HTML entity encoding) +# +# This vendored version ensures: +# - No external dependency required +# - Consistent behavior across all SDK users +# - No HTML escaping that would corrupt prompts containing < > & characters + +require_relative "mustache/mustache" diff --git a/lib/braintrust/vendor/mustache/context.rb b/lib/braintrust/vendor/mustache/context.rb new file mode 100644 index 0000000..fd65d91 --- /dev/null +++ b/lib/braintrust/vendor/mustache/context.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: Namespaced under Braintrust::Vendor + +require_relative "context_miss" + +module Braintrust + module Vendor + class Mustache + # A Context represents the context which a Mustache template is + # executed within. All Mustache tags reference keys in the Context. + class Context + # Initializes a Mustache::Context. + # + # @param [Mustache] mustache A Mustache instance. + # + def initialize(mustache) + @stack = [mustache] + @partial_template_cache = {} + end + + # A {{>partial}} tag translates into a call to the context's + # `partial` method, which would be this sucker right here. + # + # If the Mustache view handling the rendering (e.g. the view + # representing your profile page or some other template) responds + # to `partial`, we call it and render the result. + # + def partial(name, indentation = "") + # Look for the first Mustache in the stack. + mustache = mustache_in_stack + + # Indent the partial template by the given indentation. + part = mustache.partial(name).to_s.gsub(/^/, indentation) + + # Get a template object for the partial and render the result. + template_for_partial(part).render(self) + end + + def template_for_partial(partial) + @partial_template_cache[partial] ||= Template.new(partial) + end + + # Find the first Mustache in the stack. + # + # If we're being rendered inside a Mustache object as a context, + # we'll use that one. + # + # @return [Mustache] First Mustache in the stack. + # + def mustache_in_stack + @mustache_in_stack ||= @stack.find { |frame| frame.is_a?(Mustache) } + end + + # Allows customization of how Mustache escapes things. + # + # @param [Object] value Value to escape. + # + # @return [String] Escaped string. + # + def escape(value) + mustache_in_stack.escape(value) + end + + # Adds a new object to the context's internal stack. + # + # @param [Object] new_obj Object to be added to the internal stack. + # + # @return [Context] Returns the Context. + # + def push(new_obj) + @stack.unshift(new_obj) + @mustache_in_stack = nil + self + end + + # Removes the most recently added object from the context's + # internal stack. + # + # @return [Context] Returns the Context. + # + def pop + @stack.shift + @mustache_in_stack = nil + self + end + + # Can be used to add a value to the context in a hash-like way. + # + # context[:name] = "Chris" + def []=(name, value) + push(name => value) + end + + # Alias for `fetch`. + def [](name) + fetch(name, nil) + end + + # Do we know about a particular key? In other words, will calling + # `context[key]` give us a result that was set. Basically. + def has_key?(key) + fetch(key, false) + rescue ContextMiss + false + end + + # Similar to Hash#fetch, finds a value by `name` in the context's + # stack. You may specify the default return value by passing a + # second parameter. + # + # If no second parameter is passed (or raise_on_context_miss is + # set to true), will raise a ContextMiss exception on miss. + def fetch(name, default = :__raise) + @stack.each do |frame| + # Prevent infinite recursion. + next if frame == self + + value = find(frame, name, :__missing) + return value if :__missing != value + end + + if default == :__raise || mustache_in_stack.raise_on_context_miss? + raise ContextMiss.new("Can't find #{name} in #{@stack.inspect}") + else + default + end + end + + # Finds a key in an object, using whatever method is most + # appropriate. If the object is a hash, does a simple hash lookup. + # If it's an object that responds to the key as a method call, + # invokes that method. You get the idea. + # + # @param [Object] obj The object to perform the lookup on. + # @param [String,Symbol] key The key whose value you want + # @param [Object] default An optional default value, to return if the key is not found. + # + # @return [Object] The value of key in object if it is found, and default otherwise. + # + def find(obj, key, default = nil) + return find_in_hash(obj.to_hash, key, default) if obj.respond_to?(:to_hash) + + unless obj.respond_to?(key) + # no match for the key, but it may include a hyphen, so try again replacing hyphens with underscores. + key = key.to_s.tr("-", "_") + return default unless obj.respond_to?(key) + end + + meth = obj.method(key) rescue proc { obj.send(key) } + meth.arity == 1 ? meth.to_proc : meth.call + end + + def current + @stack.first + end + + private + + # Fetches a hash key if it exists, or returns the given default. + def find_in_hash(obj, key, default) + return obj[key] if obj.has_key?(key) + return obj[key.to_s] if obj.has_key?(key.to_s) + return obj[key] if obj.respond_to?(:default_proc) && obj.default_proc && obj[key] + + # If default is :__missing then we are from #fetch which is hunting through the stack + # If default is nil then we are reducing dot notation + if :__missing != default && mustache_in_stack.raise_on_context_miss? + raise ContextMiss.new("Can't find #{key} in #{obj}") + else + obj.fetch(key, default) + end + end + end + end + end +end diff --git a/lib/braintrust/vendor/mustache/context_miss.rb b/lib/braintrust/vendor/mustache/context_miss.rb new file mode 100644 index 0000000..d30b8ef --- /dev/null +++ b/lib/braintrust/vendor/mustache/context_miss.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: Namespaced under Braintrust::Vendor + +module Braintrust + module Vendor + class Mustache + # A ContextMiss is raised whenever a tag's target can not be found + # in the current context if `Mustache#raise_on_context_miss?` is + # set to true. + # + # For example, if your View class does not respond to `music` but + # your template contains a `{{music}}` tag this exception will be raised. + # + # By default it is not raised. See Mustache.raise_on_context_miss. + class ContextMiss < RuntimeError; end + end + end +end diff --git a/lib/braintrust/vendor/mustache/enumerable.rb b/lib/braintrust/vendor/mustache/enumerable.rb new file mode 100644 index 0000000..422076d --- /dev/null +++ b/lib/braintrust/vendor/mustache/enumerable.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: Namespaced under Braintrust::Vendor + +module Braintrust + module Vendor + class Mustache + Enumerable = Module.new + end + end +end diff --git a/lib/braintrust/vendor/mustache/generator.rb b/lib/braintrust/vendor/mustache/generator.rb new file mode 100644 index 0000000..c2a6b9b --- /dev/null +++ b/lib/braintrust/vendor/mustache/generator.rb @@ -0,0 +1,188 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: Namespaced under Braintrust::Vendor + +module Braintrust + module Vendor + class Mustache + # The Generator is in charge of taking an array of Mustache tokens, + # usually assembled by the Parser, and generating an interpolatable + # Ruby string. This string is considered the "compiled" template + # because at that point we're relying on Ruby to do the parsing and + # run our code. + # + # For example, let's take this template: + # + # Hi {{thing}}! + # + # If we run this through the Parser we'll get these tokens: + # + # [:multi, + # [:static, "Hi "], + # [:mustache, :etag, "thing"], + # [:static, "!\n"]] + # + # Now let's hand that to the Generator: + # + # >> puts Braintrust::Vendor::Mustache::Generator.new.compile(tokens) + # "Hi #{ctx.escape(ctx[:thing])}!\n" + class Generator + # Options can be used to manipulate the resulting ruby code string behavior. + def initialize(options = {}) + @options = options + @option_static_lambdas = options[:static_lambdas] == true + end + + # Given an array of tokens, returns an interpolatable Ruby string. + def compile(exp) + "\"#{compile!(exp)}\"" + end + + private + + # Given an array of tokens, converts them into Ruby code. In + # particular there are three types of expressions we are concerned + # with: + # + # :multi + # Mixed bag of :static, :mustache, and whatever. + # + # :static + # Normal HTML, the stuff outside of {{mustaches}}. + # + # :mustache + # Any Mustache tag, from sections to partials. + def compile!(exp) + case exp.first + when :multi + exp[1..-1].reduce(+"") { |sum, e| sum << compile!(e) } + when :static + str(exp[1]) + when :mustache + send(:"on_#{exp[1]}", *exp[2..-1]) + else + raise "Unhandled exp: #{exp.first}" + end + end + + # Callback fired when the compiler finds a section token. We're + # passed the section name and the array of tokens. + def on_section(name, offset, content, raw, delims) + # Convert the tokenized content of this section into a Ruby + # string we can use. + code = compile(content) + + # Lambda handling - default handling is to dynamically interpret + # the returned lambda result as mustache source + proc_handling = if @option_static_lambdas + <<-compiled + v.call(lambda {|v| #{code}}.call(v)).to_s + compiled + else + <<-compiled + t = Braintrust::Vendor::Mustache::Template.new(v.call(#{raw.inspect}).to_s) + def t.tokens(src=@source) + p = Braintrust::Vendor::Mustache::Parser.new + p.otag, p.ctag = #{delims.inspect} + p.compile(src) + end + t.render(ctx.dup) + compiled + end + + # Compile the Ruby for this section now that we know what's + # inside the section. + ev(<<-compiled) + case v = #{compile!(name)} + when NilClass, FalseClass + when TrueClass + #{code} + when Proc + #{proc_handling} + when Array, Enumerator, Braintrust::Vendor::Mustache::Enumerable + v.map { |_| ctx.push(_); r = #{code}; ctx.pop; r }.join + else + ctx.push(v); r = #{code}; ctx.pop; r + end + compiled + end + + # Fired when we find an inverted section. Just like `on_section`, + # we're passed the inverted section name and the array of tokens. + def on_inverted_section(name, offset, content, raw, delims) + # Convert the tokenized content of this section into a Ruby + # string we can use. + code = compile(content) + + # Compile the Ruby for this inverted section now that we know + # what's inside. + ev(<<-compiled) + v = #{compile!(name)} + if v.nil? || v == false || v.respond_to?(:empty?) && v.empty? + #{code} + end + compiled + end + + # Fired when the compiler finds a partial. We want to return code + # which calls a partial at runtime instead of expanding and + # including the partial's body to allow for recursive partials. + def on_partial(name, offset, indentation) + ev("ctx.partial(#{name.to_sym.inspect}, #{indentation.inspect})") + end + + # An unescaped tag. + def on_utag(name, offset) + ev(<<-compiled) + v = #{compile!(name)} + if v.is_a?(Proc) + v = #{@option_static_lambdas ? "v.call" : "Braintrust::Vendor::Mustache::Template.new(v.call.to_s).render(ctx.dup)"} + end + v.to_s + compiled + end + + # An escaped tag. + def on_etag(name, offset) + ev(<<-compiled) + v = #{compile!(name)} + if v.is_a?(Proc) + v = #{@option_static_lambdas ? "v.call" : "Braintrust::Vendor::Mustache::Template.new(v.call.to_s).render(ctx.dup)"} + end + ctx.escape(v) + compiled + end + + def on_fetch(names) + return "ctx.current" if names.empty? + + names = names.map { |n| n.to_sym } + + initial, *rest = names + if rest.any? + <<-compiled + #{rest.inspect}.reduce(ctx[#{initial.inspect}]) { |value, key| value && ctx.find(value, key) } + compiled + else + <<-compiled + ctx[#{initial.inspect}] + compiled + end + end + + # An interpolation-friendly version of a string, for use within a + # Ruby string. + def ev(s) + "#\{#{s}}" + end + + def str(s) + s.inspect[1..-2] + end + end + end + end +end diff --git a/lib/braintrust/vendor/mustache/mustache.rb b/lib/braintrust/vendor/mustache/mustache.rb new file mode 100644 index 0000000..7624589 --- /dev/null +++ b/lib/braintrust/vendor/mustache/mustache.rb @@ -0,0 +1,260 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: +# - Namespaced under Braintrust::Vendor +# - Disabled HTML escaping (LLM prompts don't need HTML escaping) + +require_relative "enumerable" +require_relative "template" +require_relative "context" +require_relative "settings" +require_relative "utils" + +module Braintrust + module Vendor + # Mustache is the base class from which your Mustache subclasses + # should inherit (though it can be used on its own). + # + # The typical Mustache workflow is as follows: + # + # * Create a Mustache subclass: class Stats < Mustache + # * Create a template: stats.mustache + # * Instantiate an instance: view = Stats.new + # * Render that instance: view.render + # + # You can skip the instantiation by calling `Stats.render` directly. + class Mustache + # Initialize a new Mustache instance. + # + # @param [Hash] options An options hash + # @option options [String] template_path + # @option options [String] template_extension + # @option options [String] template_file + # @option options [String] template + # @option options [String] view_namespace + # @option options [String] view_path + def initialize(options = {}) + @options = options + + initialize_settings + end + + # Instantiates an instance of this class and calls `render` with + # the passed args. + # + # @return A rendered String version of a template. + def self.render(*args) + new.render(*args) + end + + # Parses our fancy pants template file and returns normal file with + # all special {{tags}} and {{#sections}}replaced{{/sections}}. + # + # @example Render view + # @view.render("Hi {{thing}}!", :thing => :world) + # + # @example Set view template and then render + # View.template = "Hi {{thing}}!" + # @view = View.new + # @view.render(:thing => :world) + # + # @param [String,Hash] data A String template or a Hash context. + # If a Hash is given, we'll try to figure + # out the template from the class. + # @param [Hash] ctx A Hash context if `data` is a String template. + # @return [String] Returns a rendered version of a template. + def render(data = template, ctx = {}) + case data + when Hash + ctx = data + when Symbol + self.template_name = data + end + + tpl = case data + when Hash + templateify(template) + when Symbol + templateify(template) + else + templateify(data) + end + + return tpl.render(context) if ctx == {} + + begin + context.push(ctx) + tpl.render(context) + ensure + context.pop + end + end + + # Context accessors. + # + # @example Context accessors + # view = Mustache.new + # view[:name] = "Jon" + # view.template = "Hi, {{name}}!" + # view.render # => "Hi, Jon!" + def [](key) + context[key.to_sym] + end + + def []=(key, value) + context[key.to_sym] = value + end + + # A helper method which gives access to the context at a given time. + # Kind of a hack for now, but useful when you're in an iterating section + # and want access to the hash currently being iterated over. + def context + @context ||= Context.new(self) + end + + # Given a file name and an optional context, attempts to load and + # render the file as a template. + def self.render_file(name, context = {}) + render(partial(name), context) + end + + # Given a file name and an optional context, attempts to load and + # render the file as a template. + def render_file(name, context = {}) + self.class.render_file(name, context) + end + + # Given a name, attempts to read a file and return the contents as a + # string. The file is not rendered, so it might contain + # {{mustaches}}. + # + # Call `render` if you need to process it. + def self.partial(name) + new.partial(name) + end + + # Override this in your subclass if you want to do fun things like + # reading templates from a database. It will be rendered by the + # context, so all you need to do is return a string. + def partial(name) + path = "#{template_path}/#{name}.#{template_extension}" + + begin + File.read(path) + rescue + raise if raise_on_context_miss? + "" + end + end + + # BRAINTRUST MODIFICATION: No HTML escaping for LLM prompts. + # Original mustache uses CGI.escapeHTML which would turn + # characters like < > & " into HTML entities. + # For LLM prompts, we want the raw text without escaping. + # + # @param [Object] value Value to escape. + # @return [String] Unescaped content (just converted to string). + def escape(value) + value.to_s + end + + # @deprecated Use {#escape} instead. + # Kept for compatibility but also does no escaping. + def escapeHTML(str) + str.to_s + end + + # Has this instance or its class already compiled a template? + def compiled? + (@template && @template.is_a?(Template)) || self.class.compiled? + end + + private + + # When given a symbol or string representing a class, will try to produce an + # appropriate view class. + # + # @example + # Mustache.view_namespace = Hurl::Views + # Mustache.view_class(:Partial) # => Hurl::Views::Partial + def self.view_class(name) + name = classify(name.to_s) + + # Emptiness begets emptiness. + return Mustache if name.to_s.empty? + + name = "#{view_namespace}::#{name}" + const = rescued_const_get(name) + + return const if const + + const_from_file(name) + end + + def self.rescued_const_get(name) + const_get(name, true) || Mustache + rescue NameError + nil + end + + def self.const_from_file(name) + file_name = underscore(name) + file_path = "#{view_path}/#{file_name}.rb" + + return Mustache unless File.exist?(file_path) + + require file_path.chomp(".rb") + rescued_const_get(name) + end + + # Has this template already been compiled? Compilation is somewhat + # expensive so it may be useful to check this before attempting it. + def self.compiled? + @template.is_a? Template + end + + # template_partial => TemplatePartial + # template/partial => Template::Partial + def self.classify(underscored) + Mustache::Utils::String.new(underscored).classify + end + + # TemplatePartial => template_partial + # Template::Partial => template/partial + # Takes a string but defaults to using the current class' name. + def self.underscore(classified = name) + classified = superclass.name if classified.to_s.empty? + + Mustache::Utils::String.new(classified).underscore(view_namespace) + end + + # @param [Template,String] obj Turns `obj` into a template + # @param [Hash] options Options for template creation + def self.templateify(obj, options = {}) + obj.is_a?(Template) ? obj : Template.new(obj, options) + end + + def templateify(obj) + opts = {partial_resolver: method(:partial)} + opts.merge!(@options) if @options.is_a?(Hash) + self.class.templateify(obj, opts) + end + + # Return the value of the configuration setting on the superclass, or return + # the default. + # + # @param [Symbol] attr_name Name of the attribute. It should match + # the instance variable. + # @param [Object] default Default value to use if the superclass does + # not respond. + # + # @return Inherited or default configuration setting. + def self.inheritable_config_for(attr_name, default) + superclass.respond_to?(attr_name) ? superclass.send(attr_name) : default + end + end + end +end diff --git a/lib/braintrust/vendor/mustache/parser.rb b/lib/braintrust/vendor/mustache/parser.rb new file mode 100644 index 0000000..f46f3d8 --- /dev/null +++ b/lib/braintrust/vendor/mustache/parser.rb @@ -0,0 +1,364 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: Namespaced under Braintrust::Vendor + +require "strscan" + +module Braintrust + module Vendor + class Mustache + # The Parser is responsible for taking a string template and + # converting it into an array of tokens and, really, expressions. It + # raises SyntaxError if there is anything it doesn't understand and + # knows which sigil corresponds to which tag type. + # + # For example, given this template: + # + # Hi {{thing}}! + # + # Run through the Parser we'll get these tokens: + # + # [:multi, + # [:static, "Hi "], + # [:mustache, :etag, "thing"], + # [:static, "!\n"]] + class Parser + # A SyntaxError is raised when the Parser comes across unclosed + # tags, sections, illegal content in tags, or anything of that + # sort. + class SyntaxError < StandardError + def initialize(message, position) + @message = message + @lineno, @column, @line, _ = position + @stripped_line = @line.strip + @stripped_column = @column - (@line.size - @line.lstrip.size) + end + + def to_s + <<-EOF +#{@message} + Line #{@lineno} + #{@stripped_line} + #{" " * @stripped_column}^ +EOF + end + end + + # The sigil types which are valid after an opening `{{` + VALID_TYPES = ["#", "^", "/", "=", "!", "<", ">", "&", "{"].map(&:freeze) + + def self.valid_types + @valid_types ||= Regexp.new(VALID_TYPES.map { |t| Regexp.escape(t) }.join("|")) + end + + # Add a supported sigil type (with optional aliases) to the Parser. + # + # Requires a block, which will be sent the following parameters: + # + # * content - The raw content of the tag + # * fetch- A mustache context fetch expression for the content + # * padding - Indentation whitespace from the currently-parsed line + # * pre_match_position - Location of the scanner before a match was made + # + # The provided block will be evaluated against the current instance of + # Parser, and may append to the Parser's @result as needed. + def self.add_type(*types, &block) + types = types.map(&:to_s) + type, *aliases = types + method_name = :"scan_tag_#{type}" + define_method(method_name, &block) + aliases.each { |a| alias_method :"scan_tag_#{a}", method_name } + types.each { |t| VALID_TYPES << t unless VALID_TYPES.include?(t) } + @valid_types = nil + end + + # After these types of tags, all whitespace until the end of the line will + # be skipped if they are the first (and only) non-whitespace content on + # the line. + SKIP_WHITESPACE = ["#", "^", "/", "<", ">", "=", "!"].map(&:freeze) + + # The content allowed in a tag name. + ALLOWED_CONTENT = /(\w|[?!\/.=-])*/ + + # These types of tags allow any content, + # the rest only allow ALLOWED_CONTENT. + ANY_CONTENT = ["!", "="].map(&:freeze) + + attr_reader :otag, :ctag + + # Accepts an options hash which does nothing but may be used in + # the future. + def initialize(options = {}) + @options = options + @option_inline_partials_at_compile_time = options[:inline_partials_at_compile_time] + if @option_inline_partials_at_compile_time + @partial_resolver = options[:partial_resolver] + raise ArgumentError.new "Missing or invalid partial_resolver" unless @partial_resolver.respond_to? :call + end + + # Initialize default tags + self.otag ||= "{{" + self.ctag ||= "}}" + end + + # The opening tag delimiter. This may be changed at runtime. + def otag=(value) + regex = regexp value + @otag_regex = /([ \t]*)?#{regex}/ + @otag_not_regex = /(^[ \t]*)?#{regex}/ + @otag = value + end + + # The closing tag delimiter. This too may be changed at runtime. + def ctag=(value) + @ctag_regex = regexp value + @ctag = value + end + + # Given a string template, returns an array of tokens. + def compile(template) + @encoding = nil + + if template.respond_to?(:encoding) + @encoding = template.encoding + template = template.dup.force_encoding("BINARY") + end + + # Keeps information about opened sections. + @sections = [] + @result = [:multi] + @scanner = StringScanner.new(template) + + # Scan until the end of the template. + until @scanner.eos? + scan_tags || scan_text + end + + unless @sections.empty? + # We have parsed the whole file, but there's still opened sections. + type, pos, _ = @sections.pop + error "Unclosed section #{type.inspect}", pos + end + + @result + end + + private + + def content_tags(type, current_ctag_regex) + if ANY_CONTENT.include?(type) + r = /\s*#{regexp(type)}?#{current_ctag_regex}/ + scan_until_exclusive(r) + else + @scanner.scan(ALLOWED_CONTENT) + end + end + + def dispatch_based_on_type(type, content, fetch, padding, pre_match_position) + send(:"scan_tag_#{type}", content, fetch, padding, pre_match_position) + end + + def find_closing_tag(scanner, current_ctag_regex) + error "Unclosed tag" unless scanner.scan(current_ctag_regex) + end + + # Find {{mustaches}} and add them to the @result array. + def scan_tags + # Scan until we hit an opening delimiter. + start_of_line = @scanner.beginning_of_line? + pre_match_position = @scanner.pos + last_index = @result.length + + return unless @scanner.scan @otag_regex + padding = @scanner[1] || "" + + # Don't touch the preceding whitespace unless we're matching the start + # of a new line. + unless start_of_line + @result << [:static, padding] unless padding.empty? + pre_match_position += padding.length + padding = "" + end + + # Since {{= rewrites ctag, we store the ctag which should be used + # when parsing this specific tag. + current_ctag_regex = @ctag_regex + type = @scanner.scan(self.class.valid_types) + @scanner.skip(/\s*/) + + # ANY_CONTENT tags allow any character inside of them, while + # other tags (such as variables) are more strict. + content = content_tags(type, current_ctag_regex) + + # We found {{ but we can't figure out what's going on inside. + error "Illegal content in tag" if content.empty? + + fetch = [:mustache, :fetch, content.split(".")] + prev = @result + + dispatch_based_on_type(type, content, fetch, padding, pre_match_position) + + # The closing } in unescaped tags is just a hack for + # aesthetics. + type = "}" if type == "{" + + # Skip whitespace and any balancing sigils after the content + # inside this tag. + @scanner.skip(/\s+/) + @scanner.skip(regexp(type)) if type + + find_closing_tag(@scanner, current_ctag_regex) + + # If this tag was the only non-whitespace content on this line, strip + # the remaining whitespace. If not, but we've been hanging on to padding + # from the beginning of the line, re-insert the padding as static text. + if start_of_line && !@scanner.eos? + if @scanner.peek(2) =~ /\r?\n/ && SKIP_WHITESPACE.include?(type) + @scanner.skip(/\r?\n/) + else + prev.insert(last_index, [:static, padding]) unless padding.empty? + end + end + + # Store off the current scanner position now that we've closed the tag + # and consumed any irrelevant whitespace. + @sections.last[1] << @scanner.pos unless @sections.empty? + + return unless @result == [:multi] + end + + # Try to find static text, e.g. raw HTML with no {{mustaches}}. + def scan_text + text = scan_until_exclusive @otag_not_regex + + if text.nil? + # Couldn't find any otag, which means the rest is just static text. + text = @scanner.rest + # Mark as done. + @scanner.terminate + end + + text.force_encoding(@encoding) if @encoding + + @result << [:static, text] unless text.empty? + end + + # Scans the string until the pattern is matched. Returns the substring + # *excluding* the end of the match, advancing the scan pointer to that + # location. If there is no match, nil is returned. + def scan_until_exclusive(regexp) + pos = @scanner.pos + if @scanner.scan_until(regexp) + @scanner.pos -= @scanner.matched.size + @scanner.pre_match[pos..-1] + end + end + + def offset + position[0, 2] + end + + # Returns [lineno, column, line] + def position + # The rest of the current line + rest = @scanner.check_until(/\n|\Z/).to_s.chomp + + # What we have parsed so far + parsed = @scanner.string[0...@scanner.pos] + + lines = parsed.split("\n") + + [lines.size, lines.last.size - 1, lines.last + rest] + end + + # Used to quickly convert a string into a regular expression + # usable by the string scanner. + def regexp(thing) + Regexp.new Regexp.escape(thing) if thing + end + + # Raises a SyntaxError. The message should be the name of the + # error - other details such as line number and position are + # handled for you. + def error(message, pos = position) + raise SyntaxError.new(message, pos) + end + + # + # Scan tags + # + # These methods are called in `scan_tags`. Because they contain nonstandard + # characters in their method names, they are aliased to + # better named methods. + # + + # This function handles the cases where the scanned tag does not have + # a type. + def scan_tag_(content, fetch, padding, pre_match_position) + @result << [:mustache, :etag, fetch, offset] + end + + def scan_tag_block(content, fetch, padding, pre_match_position) + block = [:multi] + @result << [:mustache, :section, fetch, offset, block] + @sections << [content, position, @result] + @result = block + end + alias_method :'scan_tag_#', :scan_tag_block + + def scan_tag_inverted(content, fetch, padding, pre_match_position) + block = [:multi] + @result << [:mustache, :inverted_section, fetch, offset, block] + @sections << [content, position, @result] + @result = block + end + alias_method :'scan_tag_^', :scan_tag_inverted + + def scan_tag_close(content, fetch, padding, pre_match_position) + section, pos, result = @sections.pop + if section.nil? + error "Closing unopened #{content.inspect}" + end + + raw = @scanner.pre_match[pos[3]...pre_match_position] + padding + (@result = result).last << raw << [otag, ctag] + + if section != content + error "Unclosed section #{section.inspect}", pos + end + end + alias_method :'scan_tag_/', :scan_tag_close + + def scan_tag_comment(content, fetch, padding, pre_match_position) + end + alias_method :'scan_tag_!', :scan_tag_comment + + def scan_tag_delimiter(content, fetch, padding, pre_match_position) + self.otag, self.ctag = content.split(" ", 2) + end + alias_method :'scan_tag_=', :scan_tag_delimiter + + def scan_tag_open_partial(content, fetch, padding, pre_match_position) + @result << if @option_inline_partials_at_compile_time + partial = @partial_resolver.call content + partial.gsub!(/^/, padding) unless padding.empty? + self.class.new(@options).compile partial + else + [:mustache, :partial, content, offset, padding] + end + end + alias_method :'scan_tag_<', :scan_tag_open_partial + alias_method :'scan_tag_>', :scan_tag_open_partial + + def scan_tag_unescaped(content, fetch, padding, pre_match_position) + @result << [:mustache, :utag, fetch, offset] + end + alias_method :'scan_tag_{', :scan_tag_unescaped + alias_method :'scan_tag_&', :scan_tag_unescaped + end + end + end +end diff --git a/lib/braintrust/vendor/mustache/settings.rb b/lib/braintrust/vendor/mustache/settings.rb new file mode 100644 index 0000000..f9225ef --- /dev/null +++ b/lib/braintrust/vendor/mustache/settings.rb @@ -0,0 +1,252 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: Namespaced under Braintrust::Vendor + +# Settings which can be configured for all view classes, a single +# view class, or a single Mustache instance. +module Braintrust + module Vendor + class Mustache + def initialize_settings + @template = nil + @template_path = nil + @template_extension = nil + @template_name = nil + @template_file = nil + @raise_on_context_miss = nil + end + + def self.initialize_settings + @template = nil + @template_path = nil + @template_extension = nil + @template_name = nil + @template_file = nil + @raise_on_context_miss = nil + end + + initialize_settings + + def self.inherited(subclass) + subclass.initialize_settings + end + + # + # Template Path + # + + # The template path informs your Mustache view where to look for its + # corresponding template. By default it's the current directory (".") + # + # A class named Stat with a template_path of "app/templates" will look + # for "app/templates/stat.mustache" + + def self.template_path + @template_path ||= inheritable_config_for :template_path, "." + end + + def self.template_path=(path) + @template_path = File.expand_path(path) + @template = nil + end + + def template_path + @template_path ||= self.class.template_path + end + alias_method :path, :template_path + + def template_path=(path) + @template_path = File.expand_path(path) + @template = nil + end + + # Alias for `template_path` + def self.path + template_path + end + + # Alias for `template_path` + def self.path=(path) + self.template_path = path + end + + # + # Template Extension + # + + # A Mustache template's default extension is 'mustache', but this can be changed. + + def self.template_extension + @template_extension ||= inheritable_config_for :template_extension, "mustache" + end + + def self.template_extension=(template_extension) + @template_extension = template_extension + @template = nil + end + + def template_extension + @template_extension ||= self.class.template_extension + end + + def template_extension=(template_extension) + @template_extension = template_extension + @template = nil + end + + # + # Template Name + # + + # The template name is the Mustache template file without any + # extension or other information. Defaults to `class_name`. + # + # You may want to change this if your class is named Stat but you want + # to re-use another template. + # + # class Stat + # self.template_name = "graphs" # use graphs.mustache + # end + + def self.template_name + @template_name || underscore + end + + def self.template_name=(template_name) + @template_name = template_name + @template = nil + end + + def template_name + @template_name ||= self.class.template_name + end + + def template_name=(template_name) + @template_name = template_name + @template = nil + end + + # + # Template File + # + + # The template file is the absolute path of the file Mustache will + # use as its template. By default it's ./class_name.mustache + + def self.template_file + @template_file || "#{path}/#{template_name}.#{template_extension}" + end + + def self.template_file=(template_file) + @template_file = template_file + @template = nil + end + + # The template file is the absolute path of the file Mustache will + # use as its template. By default it's ./class_name.mustache + def template_file + @template_file || "#{path}/#{template_name}.#{template_extension}" + end + + def template_file=(template_file) + @template_file = template_file + @template = nil + end + + # + # Template + # + + # The template is the actual string Mustache uses as its template. + # There is a bit of magic here: what we get back is actually a + # Mustache::Template object, but you can still safely use `template=` + # with a string. + + def self.template + @template ||= templateify(File.read(template_file)) + end + + def self.template=(template) + @template = templateify(template) + end + + # The template can be set at the instance level. + def template + return @template if @template + + # If they sent any instance-level options use that instead of the class's. + if @template_path || @template_extension || @template_name || @template_file + @template = templateify(File.read(template_file)) + else + @template = self.class.template + end + end + + def template=(template) + @template = templateify(template) + end + + # + # Raise on context miss + # + + # Should an exception be raised when we cannot find a corresponding method + # or key in the current context? By default this is false to emulate ctemplate's + # behavior, but it may be useful to enable when debugging or developing. + # + # If set to true and there is a context miss, `Mustache::ContextMiss` will + # be raised. + + def self.raise_on_context_miss? + @raise_on_context_miss + end + + def self.raise_on_context_miss=(boolean) + @raise_on_context_miss = boolean + end + + # Instance level version of `Mustache.raise_on_context_miss?` + def raise_on_context_miss? + self.class.raise_on_context_miss? || @raise_on_context_miss + end + + def raise_on_context_miss=(boolean) + @raise_on_context_miss = boolean + end + + # + # View Namespace + # + + # The constant under which Mustache will look for views when autoloading. + # By default the view namespace is `Object`, but it might be nice to set + # it to something like `Hurl::Views` if your app's main namespace is `Hurl`. + + def self.view_namespace + @view_namespace ||= inheritable_config_for(:view_namespace, Object) + end + + def self.view_namespace=(namespace) + @view_namespace = namespace + end + + # + # View Path + # + + # Mustache searches the view path for .rb files to require when asked to find a + # view class. Defaults to "." + + def self.view_path + @view_path ||= inheritable_config_for(:view_path, ".") + end + + def self.view_path=(path) + @view_path = path + end + end + end +end diff --git a/lib/braintrust/vendor/mustache/template.rb b/lib/braintrust/vendor/mustache/template.rb new file mode 100644 index 0000000..cd8c093 --- /dev/null +++ b/lib/braintrust/vendor/mustache/template.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: Namespaced under Braintrust::Vendor + +require "cgi" + +require_relative "parser" +require_relative "generator" + +module Braintrust + module Vendor + class Mustache + # A Template represents a Mustache template. It compiles and caches + # a raw string template into something usable. + # + # The idea is this: when handed a Mustache template, convert it into + # a Ruby string by transforming Mustache tags into interpolated + # Ruby. + # + # You shouldn't use this class directly, instead: + # + # >> Braintrust::Vendor::Mustache.render(template, hash) + class Template + attr_reader :source + + # Expects a Mustache template as a string along with a template + # path, which it uses to find partials. Options may be passed. + def initialize(source, options = {}) + @source = source + @options = options + end + + # Renders the `@source` Mustache template using the given + # `context`, which should be a simple hash keyed with symbols. + # + # The first time a template is rendered, this method is overriden + # and from then on it is "compiled". Subsequent calls will skip + # the compilation step and run the Ruby version of the template + # directly. + def render(context) + # Compile our Mustache template into a Ruby string + compiled = "def render(ctx) #{compile} end" + + # Here we rewrite ourself with the interpolated Ruby version of + # our Mustache template so subsequent calls are very fast and + # can skip the compilation stage. + instance_eval(compiled, __FILE__, __LINE__ - 1) + + # Call the newly rewritten version of #render + render(context) + end + + # Does the dirty work of transforming a Mustache template into an + # interpolation-friendly Ruby string. + def compile(src = @source) + Generator.new(@options).compile(tokens(src)) + end + alias_method :to_s, :compile + + # Returns an array of tokens for a given template. + # + # @return [Array] Array of tokens. + # + def tokens(src = @source) + Parser.new(@options).compile(src) + end + + # Returns an array of tags. + # + # Tags that belong to sections will be of the form `section1.tag`. + # + # @return [Array] Returns an array of tags. + # + def tags + Template.recursor(tokens, []) do |token, section| + if [:etag, :utag].include?(token[1]) + [new_token = nil, new_section = nil, result = ((section + [token[2][2][0]]).join(".")), stop = true] + elsif [:section, :inverted_section].include?(token[1]) + [new_token = token[4], new_section = (section + [token[2][2][0]]), result = nil, stop = false] + else + [new_token = token, new_section = section, result = nil, stop = false] + end + end.flatten.reject(&:nil?).uniq + end + + # Returns an array of sections. + # + # Sections that belong to other sections will be of the form `section1.childsection` + # + # @return [Array] Returns an array of section. + # + def sections + Template.recursor(tokens, []) do |token, section| + if [:section, :inverted_section].include?(token[1]) + new_section = (section + [token[2][2][0]]) + [new_token = token[4], new_section, result = new_section.join("."), stop = false] + else + [new_token = token, new_section = section, result = nil, stop = false] + end + end.flatten.reject(&:nil?).uniq + end + + # Returns an array of partials. + # + # Partials that belong to sections are included, but the section name is not preserved + # + # @return [Array] Returns an array of partials. + # + def partials + Template.recursor(tokens, []) do |token, section| + if token[1] == :partial + [new_token = token, new_section = section, result = token[2], stop = true] + else + [new_token = token, new_section = section, result = nil, stop = false] + end + end.flatten.reject(&:nil?).uniq + end + + # Simple recursive iterator for tokens + def self.recursor(toks, section, &block) + toks.map do |token| + next unless token.is_a? Array + + if token.first == :mustache + new_token, new_section, result, stop = yield(token, section) + [result] + (stop ? [] : recursor(new_token, new_section, &block)) + else + recursor(token, section, &block) + end + end + end + end + end + end +end diff --git a/lib/braintrust/vendor/mustache/utils.rb b/lib/braintrust/vendor/mustache/utils.rb new file mode 100644 index 0000000..4975ceb --- /dev/null +++ b/lib/braintrust/vendor/mustache/utils.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Vendored from mustache gem v1.1.1 +# https://github.com/mustache/mustache +# License: MIT +# Modifications: Namespaced under Braintrust::Vendor + +module Braintrust + module Vendor + class Mustache + module Utils + class String + def initialize(string) + @string = string + end + + def classify + @string.split("/").map do |namespace| + namespace.split(/[-_]/).map do |part| + part[0] = part.chars.first.upcase + part + end.join + end.join("::") + end + + def underscore(view_namespace) + @string + .dup + .split("#{view_namespace}::") + .last + .split("::") + .map do |part| + part[0] = part[0].downcase + part.gsub(/[A-Z]/) { |s| "_" << s.downcase } + end + .join("/") + end + end + end + end + end +end diff --git a/test/braintrust/prompt_test.rb b/test/braintrust/prompt_test.rb index 247de90..935f42a 100644 --- a/test/braintrust/prompt_test.rb +++ b/test/braintrust/prompt_test.rb @@ -136,6 +136,40 @@ def test_build_handles_nested_variables assert_equal "User: Alice, Email: alice@example.com", result[:messages][0][:content] end + + def test_build_does_not_escape_html_characters + # LLM prompts should NOT have HTML escaping applied. + # Standard Mustache would turn < into < but we disable this. + data = @function_data.dup + data["prompt_data"]["prompt"]["messages"] = [ + {"role" => "user", "content" => "Write code: {{code}}"} + ] + prompt = Braintrust::Prompt.new(data) + + # These characters would be escaped by standard Mustache + code_with_html = ' & other < > stuff' + result = prompt.build(code: code_with_html) + + # Verify NO escaping occurred - raw characters preserved + assert_equal "Write code: #{code_with_html}", result[:messages][0][:content] + assert_includes result[:messages][0][:content], " & other < > stuff'}, + format: "mustache" + ) + assert_includes result, "