diff --git a/lib/braintrust.rb b/lib/braintrust.rb index 7a61327..496fc22 100644 --- a/lib/braintrust.rb +++ b/lib/braintrust.rb @@ -6,7 +6,7 @@ require_relative "braintrust/trace" require_relative "braintrust/api" require_relative "braintrust/prompt" -require_relative "braintrust/internal/experiments" +require_relative "braintrust/dataset" require_relative "braintrust/internal/env" require_relative "braintrust/eval" require_relative "braintrust/contrib" diff --git a/lib/braintrust/api.rb b/lib/braintrust/api.rb index 824c362..d808537 100644 --- a/lib/braintrust/api.rb +++ b/lib/braintrust/api.rb @@ -25,5 +25,22 @@ def datasets def functions @functions ||= API::Functions.new(self) end + + # Login to Braintrust API (idempotent) + # @return [self] + def login + @state.login + self + end + + # Generate a permalink URL to view an object in the Braintrust UI + # This is for the /object endpoint (experiments, datasets, etc.) + # For trace span permalinks, use Trace.permalink instead. + # @param object_type [String] Type of object (e.g., "experiment", "dataset") + # @param object_id [String] Object UUID + # @return [String] Permalink URL + def object_permalink(object_type:, object_id:) + @state.object_permalink(object_type: object_type, object_id: object_id) + end end end diff --git a/lib/braintrust/api/datasets.rb b/lib/braintrust/api/datasets.rb index 44553f1..120d0f7 100644 --- a/lib/braintrust/api/datasets.rb +++ b/lib/braintrust/api/datasets.rb @@ -85,7 +85,7 @@ def insert(id:, events:) # @param id [String] Dataset UUID # @return [String] Permalink URL def permalink(id:) - "#{@state.app_url}/app/#{@state.org_name}/object?object_type=dataset&object_id=#{id}" + @state.object_permalink(object_type: "dataset", object_id: id) end # Fetch records from dataset using BTQL diff --git a/lib/braintrust/api/internal/experiments.rb b/lib/braintrust/api/internal/experiments.rb new file mode 100644 index 0000000..9c09f92 --- /dev/null +++ b/lib/braintrust/api/internal/experiments.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "net/http" +require "json" +require "uri" + +module Braintrust + class API + module Internal + # Internal Experiments API + # Not part of the public API - use through Eval.run + class Experiments + def initialize(state) + @state = state + end + + # Create an experiment + # POST /v1/experiment + # @param name [String] Experiment name + # @param project_id [String] Project ID + # @param ensure_new [Boolean] If true (default), fail if exists; if false, return existing + # @param tags [Array, nil] Optional tags + # @param metadata [Hash, nil] Optional metadata + # @return [Hash] Experiment data with "id", "name", "project_id", etc. + def create(name:, project_id:, ensure_new: true, tags: nil, metadata: nil) + uri = URI("#{@state.api_url}/v1/experiment") + + payload = { + project_id: project_id, + name: name, + ensure_new: ensure_new + } + payload[:tags] = tags if tags + payload[:metadata] = metadata if metadata + + request = Net::HTTP::Post.new(uri) + request["Content-Type"] = "application/json" + request["Authorization"] = "Bearer #{@state.api_key}" + request.body = JSON.dump(payload) + + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = (uri.scheme == "https") + response = http.request(request) + + unless response.is_a?(Net::HTTPSuccess) + raise Error, "HTTP #{response.code} for POST #{uri}: #{response.body}" + end + + JSON.parse(response.body) + end + end + end + end +end diff --git a/lib/braintrust/api/internal/projects.rb b/lib/braintrust/api/internal/projects.rb new file mode 100644 index 0000000..4995a68 --- /dev/null +++ b/lib/braintrust/api/internal/projects.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require "net/http" +require "json" +require "uri" + +module Braintrust + class API + module Internal + # Internal Projects API + # Not part of the public API - use through Eval.run + class Projects + def initialize(state) + @state = state + end + + # Create or get a project by name (idempotent) + # POST /v1/project + # @param name [String] Project name + # @return [Hash] Project data with "id", "name", "org_id", etc. + def create(name:) + uri = URI("#{@state.api_url}/v1/project") + + request = Net::HTTP::Post.new(uri) + request["Content-Type"] = "application/json" + request["Authorization"] = "Bearer #{@state.api_key}" + request.body = JSON.dump({name: name}) + + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = (uri.scheme == "https") + response = http.request(request) + + unless response.is_a?(Net::HTTPSuccess) + raise Error, "HTTP #{response.code} for POST #{uri}: #{response.body}" + end + + JSON.parse(response.body) + end + end + end + end +end diff --git a/lib/braintrust/dataset.rb b/lib/braintrust/dataset.rb new file mode 100644 index 0000000..f6abb78 --- /dev/null +++ b/lib/braintrust/dataset.rb @@ -0,0 +1,185 @@ +# frozen_string_literal: true + +require_relative "api" +require_relative "internal/origin" + +module Braintrust + # High-level interface for working with Braintrust datasets. + # Provides both eager loading and lazy enumeration for efficient access to dataset records. + # + # @example Basic usage (uses global state) + # Braintrust.init(api_key: "...") + # dataset = Braintrust::Dataset.new(name: "my-dataset", project: "my-project") + # dataset.each { |record| puts record[:input] } + # + # @example With explicit API client + # api = Braintrust::API.new(state: my_state) + # dataset = Braintrust::Dataset.new(name: "my-dataset", project: "my-project", api: api) + # + # @example Eager loading for small datasets + # records = dataset.fetch_all(limit: 100) + # + # @example Using Enumerable methods + # dataset.take(10) + # dataset.select { |r| r[:tags]&.include?("important") } + # + # @example With version pinning + # dataset = Braintrust::Dataset.new(name: "my-dataset", project: "my-project", version: "1.0") + class Dataset + include Enumerable + + # Default number of records to fetch per API page + DEFAULT_PAGE_SIZE = 1000 + + attr_reader :name, :project, :version + + # Initialize a dataset reference + # @param name [String, nil] Dataset name (required if id not provided) + # @param id [String, nil] Dataset UUID (required if name not provided) + # @param project [String, nil] Project name (required if using name) + # @param version [String, nil] Optional version to pin to + # @param api [API, nil] Braintrust API client (defaults to API.new using global state) + def initialize(name: nil, id: nil, project: nil, version: nil, api: nil) + @name = name + @provided_id = id + @project = project + @version = version + @api = api || API.new + @resolved_id = nil + @metadata = nil + + validate_params! + end + + # Get the dataset ID, resolving from name if necessary + # @return [String] Dataset UUID + def id + return @provided_id if @provided_id + resolve_name! unless @resolved_id + @resolved_id + end + + # Get the dataset metadata from the API + # Makes an API call if metadata hasn't been fetched yet. + # Note: When initialized with name, metadata is fetched during name resolution. + # When initialized with ID, this triggers a separate get_by_id call. + # @return [Hash] Dataset metadata including name, description, created, etc. + def metadata + fetch_metadata! unless @metadata + @metadata + end + + # Fetch all records eagerly into an array + # @param limit [Integer, nil] Maximum records to return (nil for all) + # @return [Array] Array of records with :input, :expected, :tags, :metadata, :origin + def fetch_all(limit: nil) + records = [] + each_record(limit: limit) { |record| records << record } + records + end + + # Iterate over records lazily (implements Enumerable) + # Fetches pages on demand for memory efficiency with large datasets. + # @yield [Hash] Each record with :input, :expected, :tags, :metadata, :origin + def each(&block) + return enum_for(:each) unless block_given? + each_record(&block) + end + + private + + def validate_params! + if @provided_id.nil? && @name.nil? + raise ArgumentError, "must specify either :name or :id" + end + + if @name && @project.nil? + raise ArgumentError, ":project is required when using :name" + end + end + + # Resolve dataset name to ID (also fetches metadata as side effect) + def resolve_name! + @metadata = @api.datasets.get(project_name: @project, name: @name) + @resolved_id = @metadata["id"] + end + + # Fetch metadata explicitly (for when ID was provided directly) + def fetch_metadata! + if @provided_id + @metadata = @api.datasets.get_by_id(id: @provided_id) + else + resolve_name! unless @metadata + end + end + + # Core iteration with pagination + # @param limit [Integer, nil] Maximum records to return + def each_record(limit: nil, &block) + dataset_id = id # Resolve once + cursor = nil + count = 0 + + loop do + page_limit = if limit + [DEFAULT_PAGE_SIZE, limit - count].min + else + DEFAULT_PAGE_SIZE + end + + result = @api.datasets.fetch( + id: dataset_id, + limit: page_limit, + cursor: cursor, + version: @version + ) + + result[:records].each do |raw_record| + record = build_record(raw_record, dataset_id) + block.call(record) + count += 1 + break if limit && count >= limit + end + + # Stop if we've hit the limit or no more pages + break if limit && count >= limit + + cursor = result[:cursor] + break unless cursor + end + end + + # Build a normalized record hash from raw API response + # @param raw [Hash] Raw record from API + # @param dataset_id [String] Dataset ID for origin + # @return [Hash] Normalized record with origin + def build_record(raw, dataset_id) + record = {} + record[:input] = raw["input"] if raw.key?("input") + record[:expected] = raw["expected"] if raw.key?("expected") + record[:tags] = raw["tags"] if raw.key?("tags") + record[:metadata] = raw["metadata"] if raw.key?("metadata") + + origin = build_origin(raw, dataset_id) + record[:origin] = origin if origin + + record + end + + # Build origin JSON for tracing/linking + # @param raw [Hash] Raw record from API + # @param dataset_id [String] Dataset ID (fallback if not in record) + # @return [String, nil] JSON-serialized origin, or nil if record lacks required fields + def build_origin(raw, dataset_id) + return nil unless raw["id"] && raw["_xact_id"] + + Internal::Origin.to_json( + object_type: "dataset", + object_id: raw["dataset_id"] || dataset_id, + id: raw["id"], + xact_id: raw["_xact_id"], + created: raw["created"] + ) + end + end +end diff --git a/lib/braintrust/eval.rb b/lib/braintrust/eval.rb index 3e68f6d..0f59326 100644 --- a/lib/braintrust/eval.rb +++ b/lib/braintrust/eval.rb @@ -2,8 +2,9 @@ require_relative "eval/scorer" require_relative "eval/runner" -require_relative "internal/experiments" -require_relative "internal/origin" +require_relative "api/internal/projects" +require_relative "api/internal/experiments" +require_relative "dataset" require "opentelemetry/sdk" require "json" @@ -200,39 +201,45 @@ def scorer(name, callable = nil, &block) # @param metadata [Hash] Optional experiment metadata # @param update [Boolean] If true, allow reusing existing experiment (default: false) # @param quiet [Boolean] If true, suppress result output (default: false) - # @param state [State, nil] Braintrust state (defaults to global state) + # @param api [API, nil] Braintrust API client (defaults to API.new using global state) # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider (defaults to global) # @return [Result] def run(project:, experiment:, task:, scorers:, cases: nil, dataset: nil, parallelism: 1, tags: nil, metadata: nil, update: false, quiet: false, - state: nil, tracer_provider: nil) + api: nil, tracer_provider: nil) # Validate required parameters validate_params!(project: project, experiment: experiment, cases: cases, dataset: dataset, task: task, scorers: scorers) - # Get state from parameter or global - state ||= Braintrust.current_state - raise Error, "No state available" unless state + # Get API from parameter or create from global state + api ||= API.new - # Ensure state is logged in (to populate org_name, etc.) + # Ensure logged in (to populate org_name, etc.) # login is idempotent and returns early if already logged in - state.login + api.login # Resolve dataset to cases if dataset parameter provided if dataset - cases = resolve_dataset(dataset, project, state) + cases = resolve_dataset(dataset, project, api) end - # Register project and experiment via API - result = Internal::Experiments.get_or_create( - experiment, project, state: state, - tags: tags, metadata: metadata, update: update + # Register project and experiment via internal API + projects_api = API::Internal::Projects.new(api.state) + experiments_api = API::Internal::Experiments.new(api.state) + + project_result = projects_api.create(name: project) + experiment_result = experiments_api.create( + name: experiment, + project_id: project_result["id"], + ensure_new: !update, + tags: tags, + metadata: metadata ) - experiment_id = result[:experiment_id] - project_id = result[:project_id] - project_name = result[:project_name] + experiment_id = experiment_result["id"] + project_id = project_result["id"] + project_name = project_result["name"] # Instantiate Runner and run evaluation runner = Runner.new( @@ -242,7 +249,7 @@ def run(project:, experiment:, task:, scorers:, project_name: project_name, task: task, scorers: scorers, - state: state, + api: api, tracer_provider: tracer_provider ) result = runner.run(cases, parallelism: parallelism) @@ -286,104 +293,29 @@ def validate_params!(project:, experiment:, cases:, dataset:, task:, scorers:) end # Resolve dataset parameter to an array of case records - # @param dataset [String, Hash] Dataset specifier - # @param project [String] Project name (used as default if not specified in hash) - # @param state [State] Braintrust state + # @param dataset [String, Hash, Dataset] Dataset specifier or instance + # @param project [String] Project name (used as default if not specified) + # @param api [API] Braintrust API client # @return [Array] Array of case records - def resolve_dataset(dataset, project, state) - require_relative "api" + def resolve_dataset(dataset, project, api) + limit = nil - # Parse dataset parameter - dataset_opts = case dataset + dataset_obj = case dataset + when Dataset + dataset when String - # String: dataset name in same project - {name: dataset, project: project} + Dataset.new(name: dataset, project: project, api: api) when Hash - # Hash: explicit options - dataset.dup + opts = dataset.dup + limit = opts.delete(:limit) + opts[:project] ||= project + opts[:api] = api + Dataset.new(**opts) else - raise ArgumentError, "dataset must be String or Hash, got #{dataset.class}" + raise ArgumentError, "dataset must be String, Hash, or Dataset, got #{dataset.class}" end - # Apply defaults - dataset_opts[:project] ||= project - - # Create API client - api = API.new(state: state) - - # Resolve dataset ID - dataset_id = if dataset_opts[:id] - # ID provided directly - dataset_opts[:id] - elsif dataset_opts[:name] - # Fetch by name + project - metadata = api.datasets.get( - project_name: dataset_opts[:project], - name: dataset_opts[:name] - ) - metadata["id"] - else - raise ArgumentError, "dataset hash must specify either :name or :id" - end - - # Fetch records with pagination - limit_per_page = 1000 - max_records = dataset_opts[:limit] - version = dataset_opts[:version] - records = [] - cursor = nil - - loop do - result = api.datasets.fetch( - id: dataset_id, - limit: limit_per_page, - cursor: cursor, - version: version - ) - - records.concat(result[:records]) - - # Check if we've hit the user-specified limit - if max_records && records.length >= max_records - records = records.take(max_records) - break - end - - # Check if there's more data - cursor = result[:cursor] - break unless cursor - end - - # Filter records to only include Case-compatible fields - # Case accepts: input, expected, tags, metadata, origin - records.map do |record| - filtered = {} - filtered[:input] = record["input"] if record.key?("input") - filtered[:expected] = record["expected"] if record.key?("expected") - filtered[:tags] = record["tags"] if record.key?("tags") - filtered[:metadata] = record["metadata"] if record.key?("metadata") - - origin = build_dataset_origin(record, dataset_id) - filtered[:origin] = origin if origin - - filtered - end - end - - # Build origin JSON for a dataset record - # @param record [Hash] Record from dataset fetch API - # @param dataset_id [String] Dataset ID (fallback if not in record) - # @return [String, nil] JSON-serialized origin, or nil if record lacks required fields - def build_dataset_origin(record, dataset_id) - return nil unless record["id"] && record["_xact_id"] - - Internal::Origin.to_json( - object_type: "dataset", - object_id: record["dataset_id"] || dataset_id, - id: record["id"], - xact_id: record["_xact_id"], - created: record["created"] - ) + dataset_obj.fetch_all(limit: limit) end end end diff --git a/lib/braintrust/eval/runner.rb b/lib/braintrust/eval/runner.rb index 6040f3d..dfc1b79 100644 --- a/lib/braintrust/eval/runner.rb +++ b/lib/braintrust/eval/runner.rb @@ -18,14 +18,14 @@ class Runner MAX_PARALLELISM = Internal::ThreadPool::MAX_PARALLELISM def initialize(experiment_id:, experiment_name:, project_id:, project_name:, - task:, scorers:, state:, tracer_provider: nil) + task:, scorers:, api:, tracer_provider: nil) @experiment_id = experiment_id @experiment_name = experiment_name @project_id = project_id @project_name = project_name @task = task @scorers = normalize_scorers(scorers) - @state = state + @api = api @tracer_provider = tracer_provider || OpenTelemetry.tracer_provider @tracer = @tracer_provider.tracer("braintrust-eval") @parent_attr = "experiment_id:#{experiment_id}" @@ -61,7 +61,7 @@ def run(cases, parallelism: 1) duration = Time.now - start_time # Generate permalink - permalink = "#{state.app_url}/app/#{state.org_name}/object?object_type=experiment&object_id=#{experiment_id}" + permalink = @api.object_permalink(object_type: "experiment", object_id: experiment_id) Result.new( experiment_id: experiment_id, @@ -78,7 +78,7 @@ def run(cases, parallelism: 1) private attr_reader :experiment_id, :experiment_name, :project_id, :project_name, - :task, :scorers, :state, :tracer, :parent_attr + :task, :scorers, :tracer, :parent_attr # Run a single test case with OpenTelemetry tracing # Creates eval span (parent) with task and score as children diff --git a/lib/braintrust/internal/experiments.rb b/lib/braintrust/internal/experiments.rb deleted file mode 100644 index a5a64a1..0000000 --- a/lib/braintrust/internal/experiments.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true - -require "net/http" -require "json" -require "uri" -require_relative "../logger" - -module Braintrust - module Internal - # Experiments module provides internal API methods for registering projects and experiments - # Methods are marked private to prevent direct user access - use through Eval.run - module Experiments - # Public convenience method to register/get both project and experiment - # @param experiment_name [String] The experiment name - # @param project_name [String] The project name - # @param state [State] Braintrust state with API key and URL - # @param tags [Array, nil] Optional experiment tags - # @param metadata [Hash, nil] Optional experiment metadata - # @param update [Boolean] If true, allow reusing existing experiment (default: false) - # @return [Hash] Hash with :experiment_id, :experiment_name, :project_id, :project_name - def self.get_or_create(experiment_name, project_name, state:, - tags: nil, metadata: nil, update: false) - # Register/get project first - project = register_project(project_name, state) - - # Then register/get experiment - experiment = register_experiment( - experiment_name, - project["id"], - state, - tags: tags, - metadata: metadata, - update: update - ) - - { - experiment_id: experiment["id"], - experiment_name: experiment["name"], - project_id: project["id"], - project_name: project["name"] - } - end - - # Register or get a project by name - # POST /v1/project with {name: "project-name"} - # Returns existing project if already exists - # @param name [String] Project name - # @param state [State] Braintrust state - # @return [Hash] Project data with "id", "name", "org_id", etc. - # @raise [Braintrust::Error] if API call fails - def self.register_project(name, state) - Log.debug("Registering project: #{name}") - - uri = URI("#{state.api_url}/v1/project") - request = Net::HTTP::Post.new(uri) - request["Content-Type"] = "application/json" - request["Authorization"] = "Bearer #{state.api_key}" - request.body = JSON.dump({name: name}) - - http = Net::HTTP.new(uri.hostname, uri.port) - http.use_ssl = true if uri.scheme == "https" - - response = http.start do |http_session| - http_session.request(request) - end - - Log.debug("Register project response: [#{response.code}]") - - # Handle response codes - unless response.is_a?(Net::HTTPSuccess) - raise Error, "Failed to register project '#{name}': [#{response.code}] #{response.body}" - end - - project = JSON.parse(response.body) - Log.debug("Project registered: #{project["id"]} (#{project["name"]})") - project - end - private_class_method :register_project - - # Register or get an experiment by name - # POST /v1/experiment with {project_id:, name:, ensure_new:, tags:[], metadata:{}} - # @param name [String] Experiment name - # @param project_id [String] Project ID - # @param state [State] Braintrust state - # @param tags [Array, nil] Optional tags - # @param metadata [Hash, nil] Optional metadata - # @param update [Boolean] If true, allow reusing existing experiment (ensure_new: false) - # @return [Hash] Experiment data with "id", "name", "project_id", etc. - # @raise [Braintrust::Error] if API call fails - def self.register_experiment(name, project_id, state, tags: nil, metadata: nil, update: false) - Log.debug("Registering experiment: #{name} (project: #{project_id}, update: #{update})") - - uri = URI("#{state.api_url}/v1/experiment") - request = Net::HTTP::Post.new(uri) - request["Content-Type"] = "application/json" - request["Authorization"] = "Bearer #{state.api_key}" - - payload = { - project_id: project_id, - name: name, - ensure_new: !update # When update=true, allow reusing existing experiment - } - payload[:tags] = tags if tags - payload[:metadata] = metadata if metadata - - request.body = JSON.dump(payload) - - http = Net::HTTP.new(uri.hostname, uri.port) - http.use_ssl = true if uri.scheme == "https" - - response = http.start do |http_session| - http_session.request(request) - end - - Log.debug("Register experiment response: [#{response.code}]") - - # Handle response codes - unless response.is_a?(Net::HTTPSuccess) - raise Error, "Failed to register experiment '#{name}': [#{response.code}] #{response.body}" - end - - experiment = JSON.parse(response.body) - Log.debug("Experiment registered: #{experiment["id"]} (#{experiment["name"]})") - experiment - end - private_class_method :register_experiment - end - end -end diff --git a/lib/braintrust/state.rb b/lib/braintrust/state.rb index a56181e..f8c6275 100644 --- a/lib/braintrust/state.rb +++ b/lib/braintrust/state.rb @@ -139,6 +139,16 @@ def login end end + # Generate a permalink URL to view an object in the Braintrust UI + # This is for the /object endpoint (experiments, datasets, etc.) + # For trace span permalinks, use Trace.permalink instead. + # @param object_type [String] Type of object (e.g., "experiment", "dataset") + # @param object_id [String] Object UUID + # @return [String] Permalink URL + def object_permalink(object_type:, object_id:) + "#{@app_url}/app/#{@org_name}/object?object_type=#{object_type}&object_id=#{object_id}" + end + # Login to Braintrust API in a background thread with retry logic # Retries indefinitely with exponential backoff until success # Idempotent: returns early if already logged in diff --git a/test/braintrust/dataset_test.rb b/test/braintrust/dataset_test.rb new file mode 100644 index 0000000..8ad15f7 --- /dev/null +++ b/test/braintrust/dataset_test.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +require "test_helper" +require "braintrust/dataset" + +class Braintrust::DatasetTest < Minitest::Test + # ============================================ + # Initialization tests + # ============================================ + + def test_initialize_with_name_and_project + api = mock_api + dataset = Braintrust::Dataset.new(name: "my-dataset", project: "my-project", api: api) + + assert_equal "my-dataset", dataset.name + assert_equal "my-project", dataset.project + end + + def test_initialize_with_id + api = mock_api + dataset = Braintrust::Dataset.new(id: "dataset-123", api: api) + + assert_equal "dataset-123", dataset.id + end + + def test_initialize_requires_name_or_id + api = mock_api + error = assert_raises(ArgumentError) do + Braintrust::Dataset.new(api: api) + end + + assert_match(/must specify either :name or :id/, error.message) + end + + def test_initialize_requires_project_when_using_name + api = mock_api + error = assert_raises(ArgumentError) do + Braintrust::Dataset.new(name: "my-dataset", api: api) + end + + assert_match(/:project is required when using :name/, error.message) + end + + def test_initialize_with_version + api = mock_api + dataset = Braintrust::Dataset.new(name: "my-dataset", project: "my-project", version: "1.0", api: api) + + assert_equal "1.0", dataset.version + end + + def test_initialize_defaults_api_from_global_state + # Set up global state + state = get_unit_test_state + Braintrust::State.global = state + + dataset = Braintrust::Dataset.new(name: "my-dataset", project: "my-project") + + assert_equal "my-dataset", dataset.name + ensure + Braintrust::State.global = nil + end + + # ============================================ + # ID resolution tests + # ============================================ + + def test_id_returns_provided_id_directly + api = mock_api + dataset = Braintrust::Dataset.new(id: "dataset-123", api: api) + + # Should return ID without making API calls + assert_equal "dataset-123", dataset.id + end + + # ============================================ + # Enumerable tests + # ============================================ + + def test_dataset_is_enumerable + api = mock_api + dataset = Braintrust::Dataset.new(id: "dataset-123", api: api) + + assert dataset.respond_to?(:each) + assert dataset.respond_to?(:map) + assert dataset.respond_to?(:select) + assert dataset.respond_to?(:take) + end + + # ============================================ + # Origin generation tests + # ============================================ + + def test_build_origin_creates_valid_json + api = mock_api + dataset = Braintrust::Dataset.new(id: "dataset-123", api: api) + + raw_record = { + "id" => "record-456", + "_xact_id" => "1000196022104685824", + "dataset_id" => "dataset-123", + "created" => "2025-10-24T15:29:18.118Z", + "input" => "test" + } + + origin = dataset.send(:build_origin, raw_record, "dataset-123") + + assert origin + parsed = JSON.parse(origin) + assert_equal "dataset", parsed["object_type"] + assert_equal "dataset-123", parsed["object_id"] + assert_equal "record-456", parsed["id"] + assert_equal "1000196022104685824", parsed["_xact_id"] + end + + def test_build_origin_uses_fallback_dataset_id + api = mock_api + dataset = Braintrust::Dataset.new(id: "dataset-123", api: api) + + raw_record = { + "id" => "record-456", + "_xact_id" => "1000196022104685824" + # No dataset_id in record + } + + origin = dataset.send(:build_origin, raw_record, "fallback-id") + + parsed = JSON.parse(origin) + assert_equal "fallback-id", parsed["object_id"] + end + + def test_build_origin_returns_nil_when_missing_required_fields + api = mock_api + dataset = Braintrust::Dataset.new(id: "dataset-123", api: api) + + # Missing id + record_no_id = {"_xact_id" => "123"} + assert_nil dataset.send(:build_origin, record_no_id, "dataset-id") + + # Missing _xact_id + record_no_xact = {"id" => "record-123"} + assert_nil dataset.send(:build_origin, record_no_xact, "dataset-id") + end + + # ============================================ + # Integration tests with VCR + # ============================================ + + def test_fetch_all_returns_records_with_origin + VCR.use_cassette("dataset/fetch_all") do + state = get_integration_test_state + api = Braintrust::API.new(state: state) + + # Create/reuse test dataset + project_name = "ruby-sdk-test" + dataset_name = "test-ruby-sdk-dataset-fetch" + + result = api.datasets.create( + name: dataset_name, + project_name: project_name + ) + dataset_id = result["dataset"]["id"] + + # Insert test record + api.datasets.insert( + id: dataset_id, + events: [{input: "fetch-test", expected: "FETCH-TEST"}] + ) + + # Use Dataset class to fetch + dataset = Braintrust::Dataset.new(name: dataset_name, project: project_name, api: api) + records = dataset.fetch_all(limit: 1) + + assert records.any?, "Expected at least one record" + + record = records.first + assert record[:input], "Record should have input" + assert record[:origin], "Record should have origin" + + # Verify origin structure + origin = JSON.parse(record[:origin]) + assert_equal "dataset", origin["object_type"] + end + end + + def test_each_iterates_lazily + VCR.use_cassette("dataset/each_lazy") do + state = get_integration_test_state + api = Braintrust::API.new(state: state) + + # Create/reuse test dataset + project_name = "ruby-sdk-test" + dataset_name = "test-ruby-sdk-dataset-lazy" + + result = api.datasets.create( + name: dataset_name, + project_name: project_name + ) + dataset_id = result["dataset"]["id"] + + # Insert test records + api.datasets.insert( + id: dataset_id, + events: [ + {input: "lazy-1", expected: "LAZY-1"}, + {input: "lazy-2", expected: "LAZY-2"} + ] + ) + + # Use take to only fetch what we need + dataset = Braintrust::Dataset.new(name: dataset_name, project: project_name, api: api) + first_record = dataset.take(1).first + + assert first_record, "Expected to get first record" + assert first_record[:input], "Record should have input" + end + end + + private + + def mock_api + # Create a minimal mock API that won't make real calls + Minitest::Mock.new + end +end diff --git a/test/braintrust/eval/functions_test.rb b/test/braintrust/eval/functions_test.rb index cef3e91..d985f21 100644 --- a/test/braintrust/eval/functions_test.rb +++ b/test/braintrust/eval/functions_test.rb @@ -181,7 +181,7 @@ def test_use_remote_task_in_eval_run output.to_s.include?(expected) ? 1.0 : 0.0 end ], - state: state, + api: api, quiet: true ) @@ -246,7 +246,7 @@ def test_use_remote_scorer_in_eval_run ], task: task, scorers: [scorer], - state: state, + api: api, quiet: true ) diff --git a/test/braintrust/eval/runner_test.rb b/test/braintrust/eval/runner_test.rb index 570a8fe..09d11e5 100644 --- a/test/braintrust/eval/runner_test.rb +++ b/test/braintrust/eval/runner_test.rb @@ -24,7 +24,7 @@ def test_runner_run_returns_result_object project_name: "test-project", task: ->(input) { input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| (o == e) ? 1.0 : 0.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -43,7 +43,7 @@ def test_runner_run_populates_result_fields project_name: "test-project", task: ->(input) { input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -71,7 +71,7 @@ def test_runner_run_generates_correct_permalink project_name: "test-project", task: ->(input) { input }, scorers: [Braintrust::Eval.scorer("test") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -95,7 +95,7 @@ def test_runner_run_executes_task_for_each_case input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -127,7 +127,7 @@ def test_runner_run_executes_scorers_with_correct_args 1.0 } ], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -156,7 +156,7 @@ def test_runner_run_accepts_array_of_hashes project_name: "test-project", task: ->(input) { input }, scorers: [Braintrust::Eval.scorer("test") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -174,7 +174,7 @@ def test_runner_run_accepts_cases_object project_name: "test-project", task: ->(input) { input }, scorers: [Braintrust::Eval.scorer("test") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -193,7 +193,7 @@ def test_runner_run_accepts_enumerable project_name: "test-project", task: ->(input) { input }, scorers: [Braintrust::Eval.scorer("test") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -220,7 +220,7 @@ def test_runner_run_collects_task_errors input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -250,7 +250,7 @@ def test_runner_run_collects_scorer_errors 1.0 } ], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -279,7 +279,7 @@ def test_runner_run_continues_after_task_error input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -307,7 +307,7 @@ def test_runner_run_collects_multiple_errors input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -339,7 +339,7 @@ def test_runner_run_with_parallelism_greater_than_1 input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -368,7 +368,7 @@ def test_runner_run_sequential_preserves_order input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -395,7 +395,7 @@ def test_runner_run_default_parallelism_is_sequential input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -419,7 +419,7 @@ def test_runner_run_creates_eval_spans project_name: "test-project", task: ->(input) { input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -442,7 +442,7 @@ def test_runner_run_creates_task_spans project_name: "test-project", task: ->(input) { input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -465,7 +465,7 @@ def test_runner_run_creates_score_spans project_name: "test-project", task: ->(input) { input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -491,7 +491,7 @@ def test_runner_run_records_scores_on_span Braintrust::Eval.scorer("accuracy") { |i, e, o| 0.95 }, Braintrust::Eval.scorer("relevance") { |i, e, o| 0.87 } ], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -526,7 +526,7 @@ def test_runner_can_be_run_multiple_times input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -551,7 +551,7 @@ def test_runner_runs_are_independent input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| 1.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -575,7 +575,7 @@ def test_runner_scores_is_reset_between_runs project_name: "test-project", task: ->(input) { input.upcase }, scorers: [Braintrust::Eval.scorer("exact") { |i, e, o| (o == e) ? 1.0 : 0.0 }], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) diff --git a/test/braintrust/eval_test.rb b/test/braintrust/eval_test.rb index ff28e7e..17511f8 100644 --- a/test/braintrust/eval_test.rb +++ b/test/braintrust/eval_test.rb @@ -16,7 +16,7 @@ def test_eval_scorer_helper def test_eval_run_basic VCR.use_cassette("eval/run_basic") do - state = get_integration_test_state + api = get_integration_test_api task = ->(input) { input.upcase } scorer = Braintrust::Eval.scorer("exact") do |input, expected, output| @@ -32,7 +32,7 @@ def test_eval_run_basic ], task: task, scorers: [scorer], - state: state, + api: api, quiet: true ) @@ -45,7 +45,7 @@ def test_eval_run_basic def test_eval_run_with_task_error VCR.use_cassette("eval/run_task_error") do - state = get_integration_test_state + api = get_integration_test_api task = ->(input) { raise "Task failed!" if input == "bad" @@ -65,7 +65,7 @@ def test_eval_run_with_task_error ], task: task, scorers: [scorer], - state: state, + api: api, quiet: true ) @@ -77,7 +77,7 @@ def test_eval_run_with_task_error def test_eval_run_with_scorer_error VCR.use_cassette("eval/run_scorer_error") do - state = get_integration_test_state + api = get_integration_test_api task = ->(input) { input.upcase } @@ -95,7 +95,7 @@ def test_eval_run_with_scorer_error ], task: task, scorers: [scorer], - state: state, + api: api, quiet: true ) @@ -125,7 +125,7 @@ def test_eval_scorer_error_records_exception_event cases: [{input: "bad", expected: "BAD"}], task: task, scorers: [good_scorer, failing_scorer], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -149,7 +149,7 @@ def test_eval_scorer_error_records_exception_event def test_eval_run_with_multiple_scorers VCR.use_cassette("eval/run_multiple_scorers") do - state = get_integration_test_state + api = get_integration_test_api task = ->(input) { input.upcase } @@ -169,7 +169,7 @@ def test_eval_run_with_multiple_scorers ], task: task, scorers: [scorer1, scorer2], - state: state, + api: api, quiet: true ) @@ -179,7 +179,7 @@ def test_eval_run_with_multiple_scorers def test_eval_run_with_callable_task VCR.use_cassette("eval/run_callable_task") do - state = get_integration_test_state + api = get_integration_test_api callable_task = Class.new do def call(input) @@ -199,7 +199,7 @@ def call(input) ], task: callable_task, scorers: [scorer], - state: state, + api: api, quiet: true ) @@ -220,16 +220,14 @@ def test_eval_run_validates_required_params def test_eval_run_validates_task_callable # Test that task must be callable (no API call needed) - state = get_unit_test_state - + # Note: Validation happens before API is used, so we can pass nil error = assert_raises(ArgumentError) do Braintrust::Eval.run( project: "test", experiment: "test", cases: [], task: "not callable", # String is not callable - scorers: [], - state: state + scorers: [] ) end @@ -238,7 +236,7 @@ def test_eval_run_validates_task_callable def test_eval_run_with_method_scorer VCR.use_cassette("eval/run_method_scorer") do - state = get_integration_test_state + api = get_integration_test_api task = ->(input) { input.upcase } # Use a lambda instead of nested method @@ -252,7 +250,7 @@ def test_eval_run_with_method_scorer ], task: task, scorers: [test_method_scorer], # Pass lambda directly - state: state, + api: api, quiet: true ) @@ -279,7 +277,7 @@ def test_eval_task_error_records_exception_on_task_span cases: [{input: "bad", expected: "BAD"}], task: task, scorers: [scorer], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -308,7 +306,7 @@ def test_eval_run_with_tracing rig = setup_otel_test_rig # Initialize and login - state = get_integration_test_state + api = get_integration_test_api task = ->(input) { input.upcase } scorer = Braintrust::Eval.scorer("exact") { |i, e, o| (o == e) ? 1.0 : 0.0 } @@ -319,7 +317,7 @@ def test_eval_run_with_tracing cases: [{input: "hello", expected: "HELLO"}], task: task, scorers: [scorer], - state: state, + api: api, tracer_provider: rig.tracer_provider, quiet: true ) @@ -367,8 +365,7 @@ def test_eval_run_with_tracing # Test dataset integration: dataset as string (same project as experiment) def test_eval_run_with_dataset_string VCR.use_cassette("eval/dataset_string") do - state = get_integration_test_state - api = Braintrust::API.new(state: state) + api = get_integration_test_api # Create a test dataset with records project_name = "ruby-sdk-test" @@ -403,7 +400,7 @@ def test_eval_run_with_dataset_string dataset: dataset_name, # String - should fetch from same project task: task, scorers: [scorer], - state: state, + api: api, quiet: true ) @@ -417,8 +414,7 @@ def test_eval_run_with_dataset_string # Test dataset integration: dataset as hash with name + project def test_eval_run_with_dataset_hash_name_project VCR.use_cassette("eval/dataset_hash_name_project") do - state = get_integration_test_state - api = Braintrust::API.new(state: state) + api = get_integration_test_api # Create a test dataset project_name = "ruby-sdk-test" @@ -446,7 +442,7 @@ def test_eval_run_with_dataset_hash_name_project dataset: {name: dataset_name, project: project_name}, task: task, scorers: [scorer], - state: state, + api: api, quiet: true ) @@ -457,8 +453,7 @@ def test_eval_run_with_dataset_hash_name_project # Test dataset integration: dataset as hash with id def test_eval_run_with_dataset_hash_id VCR.use_cassette("eval/dataset_hash_id") do - state = get_integration_test_state - api = Braintrust::API.new(state: state) + api = get_integration_test_api # Create a test dataset project_name = "ruby-sdk-test" @@ -486,7 +481,7 @@ def test_eval_run_with_dataset_hash_id dataset: {id: dataset_id}, # By ID only task: task, scorers: [scorer], - state: state, + api: api, quiet: true ) @@ -497,8 +492,7 @@ def test_eval_run_with_dataset_hash_id # Test dataset integration: dataset with limit option def test_eval_run_with_dataset_limit VCR.use_cassette("eval/dataset_limit") do - state = get_integration_test_state - api = Braintrust::API.new(state: state) + api = get_integration_test_api # Create a test dataset with multiple records project_name = "ruby-sdk-test" @@ -537,7 +531,7 @@ def test_eval_run_with_dataset_limit dataset: {name: dataset_name, project: project_name, limit: 2}, task: task, scorers: [scorer], - state: state, + api: api, quiet: true ) @@ -549,7 +543,7 @@ def test_eval_run_with_dataset_limit # Test dataset integration: error when both dataset and cases provided def test_eval_run_with_both_dataset_and_cases_errors VCR.use_cassette("eval/run_both_dataset_and_cases_error") do - state = get_integration_test_state + api = get_integration_test_api task = ->(input) { input.upcase } scorer = Braintrust::Eval.scorer("exact") { |i, e, o| (o == e) ? 1.0 : 0.0 } @@ -563,7 +557,7 @@ def test_eval_run_with_both_dataset_and_cases_errors cases: [{input: "test"}], task: task, scorers: [scorer], - state: state + api: api ) end @@ -599,7 +593,7 @@ def test_eval_run_with_parallelism_executes_all_cases task: task, scorers: [scorer], parallelism: 3, - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -635,7 +629,7 @@ def test_eval_run_parallelism_1_matches_sequential task: task, scorers: [scorer], parallelism: 1, - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -666,7 +660,7 @@ def test_eval_run_parallel_collects_errors_from_threads task: task, scorers: [scorer], parallelism: 3, - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -692,7 +686,7 @@ def test_eval_run_parallelism_exceeds_max_raises task: task, scorers: [scorer], parallelism: max_parallelism + 1, - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) end @@ -724,7 +718,7 @@ def test_eval_run_invalid_parallelism_falls_back_to_sequential task: task, scorers: [scorer], parallelism: 0, - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) assert result.success? @@ -744,7 +738,7 @@ def test_eval_run_invalid_parallelism_falls_back_to_sequential task: task, scorers: [scorer], parallelism: -1, - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) assert result.success? @@ -757,30 +751,6 @@ def test_eval_run_invalid_parallelism_falls_back_to_sequential # Origin is automatically generated when using remote datasets. # It links eval spans back to their source dataset records in the UI. - def test_build_dataset_origin_uses_fallback_dataset_id - # Some API responses may not include dataset_id in the record itself - record = { - "id" => "record-123", - "_xact_id" => "1000196022104685824", - "created" => "2025-10-24T15:29:18.118Z" - } - - origin = Braintrust::Eval.send(:build_dataset_origin, record, "fallback-dataset-id") - - parsed = JSON.parse(origin) - assert_equal "fallback-dataset-id", parsed["object_id"] - end - - def test_build_dataset_origin_returns_nil_when_missing_required_fields - # Missing id - can't link to a specific record - record_no_id = {"_xact_id" => "1000196022104685824"} - assert_nil Braintrust::Eval.send(:build_dataset_origin, record_no_id, "dataset-id") - - # Missing _xact_id - can't identify the transaction - record_no_xact = {"id" => "record-123"} - assert_nil Braintrust::Eval.send(:build_dataset_origin, record_no_xact, "dataset-id") - end - def test_runner_does_not_set_origin_when_case_has_no_origin # Inline cases (not from remote datasets) have no origin rig = setup_otel_test_rig @@ -796,7 +766,7 @@ def test_runner_does_not_set_origin_when_case_has_no_origin cases: [{input: "hello", expected: "HELLO"}], task: task, scorers: [scorer], - state: rig.state, + api: rig.api, tracer_provider: rig.tracer_provider ) @@ -813,10 +783,8 @@ def test_eval_with_remote_dataset_sets_origin_from_api_response VCR.use_cassette("eval/dataset_origin") do # Set up span capture (uses unit test state internally, but we override state for API calls) rig = setup_otel_test_rig - # Get integration state for real API calls via VCR - state = get_integration_test_state - - api = Braintrust::API.new(state: state) + # Get integration API for real API calls via VCR + api = get_integration_test_api # Create/reuse test dataset (idempotent) project_name = "ruby-sdk-test" @@ -845,7 +813,7 @@ def test_eval_with_remote_dataset_sets_origin_from_api_response dataset: dataset_name, task: task, scorers: [scorer], - state: state, + api: api, tracer_provider: rig.tracer_provider, quiet: true ) diff --git a/test/braintrust/internal/experiments_test.rb b/test/braintrust/internal/experiments_test.rb deleted file mode 100644 index ef4d11f..0000000 --- a/test/braintrust/internal/experiments_test.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" -require "braintrust/internal/experiments" - -class Braintrust::Internal::ExperimentsTest < Minitest::Test - def test_get_or_create_basic - VCR.use_cassette("experiments/get_or_create_basic") do - state = get_integration_test_state - - result = Braintrust::Internal::Experiments.get_or_create( - "test-ruby-sdk-experiment-basic", - "ruby-sdk-test", - state: state - ) - - assert result[:experiment_id] - assert result[:experiment_name] - assert result[:project_id] - assert_equal "ruby-sdk-test", result[:project_name] - end - end - - def test_get_or_create_with_tags_and_metadata - VCR.use_cassette("experiments/get_or_create_with_tags") do - state = get_integration_test_state - - result = Braintrust::Internal::Experiments.get_or_create( - "test-ruby-sdk-experiment-tags", - "ruby-sdk-test", - state: state, - tags: ["test", "ruby"], - metadata: {version: "1.0", author: "claude"} - ) - - assert result[:experiment_id] - assert result[:project_id] - end - end - - def test_get_or_create_with_update_flag - VCR.use_cassette("experiments/get_or_create_with_update") do - state = get_integration_test_state - - # First create with update: false (new experiment) - result1 = Braintrust::Internal::Experiments.get_or_create( - "test-experiment-update", - "ruby-sdk-test", - state: state, - update: false - ) - - # Then with update: true (should allow reusing) - result2 = Braintrust::Internal::Experiments.get_or_create( - "test-experiment-update", - "ruby-sdk-test", - state: state, - update: true - ) - - # Both should succeed and return experiment IDs - assert result1[:experiment_id] - assert result2[:experiment_id] - end - end - - def test_register_project_is_private - # Test that register_project is private and cannot be called directly - error = assert_raises(NoMethodError) do - Braintrust::Internal::Experiments.register_project("test", nil) - end - - assert_match(/private method|undefined method/, error.message) - end - - def test_register_experiment_is_private - # Test that register_experiment is private and cannot be called directly - error = assert_raises(NoMethodError) do - Braintrust::Internal::Experiments.register_experiment("test", "proj_id", nil) - end - - assert_match(/private method|undefined method/, error.message) - end -end diff --git a/test/fixtures/vcr_cassettes/dataset/each_lazy.yml b/test/fixtures/vcr_cassettes/dataset/each_lazy.yml new file mode 100644 index 0000000..7490df6 --- /dev/null +++ b/test/fixtures/vcr_cassettes/dataset/each_lazy.yml @@ -0,0 +1,342 @@ +--- +http_interactions: +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '395' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-OTllZTU4NGYtZmVhNy00NWQ2LTk4N2MtOWM5NDFlY2Y4MWRi'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 04 Feb 2026 21:02:20 GMT + Etag: + - '"12n7ok4b5phaz"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - OTllZTU4NGYtZmVhNy00NWQ2LTk4N2MtOWM5NDFlY2Y4MWRi + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - cle1::iad1::dhhxm-1770238939945-4bd333d28aba + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Wed, 04 Feb 2026 21:02:20 GMT +- request: + method: post + uri: https://www.braintrust.dev/api/dataset/register + body: + encoding: UTF-8 + string: '{"dataset_name":"test-ruby-sdk-dataset-lazy","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_name":"ruby-sdk-test"}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '613' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MjNkNjcyMjEtMjIzOC00MWUxLWFjZjktYjM5MjVhNWM5ZGU0'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 04 Feb 2026 21:02:20 GMT + Etag: + - '"pn5e5w8hn0h1"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/dataset/register" + X-Nonce: + - MjNkNjcyMjEtMjIzOC00MWUxLWFjZjktYjM5MjVhNWM5ZGU0 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - cle1::iad1::fnljg-1770238940113-fd2bfecfa1ea + body: + encoding: UTF-8 + string: '{"project":{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","description":null,"created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null},"dataset":{"id":"e3cb8850-b409-4a34-a12c-89f93e5331c3","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-lazy","description":null,"created":"2026-02-04T21:02:20.204Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"url_slug":"test-ruby-sdk-dataset-lazy"},"found_existing":false}' + recorded_at: Wed, 04 Feb 2026 21:02:20 GMT +- request: + method: post + uri: https://staging-api.braintrust.dev/v1/dataset/e3cb8850-b409-4a34-a12c-89f93e5331c3/insert + body: + encoding: UTF-8 + string: '{"events":[{"input":"lazy-1","expected":"LAZY-1"},{"input":"lazy-2","expected":"LAZY-2"}]}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '91' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - ORD56-P14 + - ORD58-P11 + Date: + - Wed, 04 Feb 2026 21:02:20 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 1f3eb747-abaf-436a-9b0b-56ad40d140e5 + X-Bt-Internal-Trace-Id: + - 6983b3dc0000000056909d088cf32ccd + X-Amz-Apigw-Id: + - YRkKfFEzIAMEnZw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"5b-fltfYciuLRqjr6hEaGalGnQoq68" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-6983b3dc-5f4972ea0bbde5e8417bb472;Parent=59672849e017b3cd;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 c41d7e84814c40081afb134b22aff29a.cloudfront.net (CloudFront), 1.1 3687dc2c690d41a17862d629e7e00b90.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - DvSG9JsB6BagbD4rT27XY0uICoAtlocIQOsZKCYsuRoSDk41Ko83Fg== + body: + encoding: ASCII-8BIT + string: '{"row_ids":["3c537c61-7129-496a-b304-2e6a62dfc45b","734e8320-175f-4240-a2f8-26c42b56e8ed"]}' + recorded_at: Wed, 04 Feb 2026 21:02:20 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/v1/dataset?dataset_name=test-ruby-sdk-dataset-lazy&project_name=ruby-sdk-test + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '326' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - ORD56-P14 + - ORD58-P11 + Date: + - Wed, 04 Feb 2026 21:02:20 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 503b01da-9b5a-4925-9d18-588225b81f7c + X-Bt-Internal-Trace-Id: + - 6983b3dc0000000007fafa80a170f0a6 + X-Amz-Apigw-Id: + - YRkKjHpNoAMEBfA= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"146-qK8LiUudQSDvyStGdyQsK7lGEQU" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-6983b3dc-2cafedf91ca40bb82adb4f4a;Parent=141341e8795c031c;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 bf89839dd141364aec15e919e53bb54a.cloudfront.net (CloudFront), 1.1 1fac415a93bf9552cac3e94ae1ced256.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 4xWOOA2Q_7rUZLk_mTKgy_eM4Lfc0-45y8mKu8ixmkzL7cr03Wi7XA== + body: + encoding: ASCII-8BIT + string: '{"objects":[{"id":"e3cb8850-b409-4a34-a12c-89f93e5331c3","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-lazy","description":null,"created":"2026-02-04T21:02:20.204Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"url_slug":"test-ruby-sdk-dataset-lazy"}]}' + recorded_at: Wed, 04 Feb 2026 21:02:20 GMT +- request: + method: post + uri: https://staging-api.braintrust.dev/btql + body: + encoding: UTF-8 + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"e3cb8850-b409-4a34-a12c-89f93e5331c3"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Content-Length: + - '1248' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - ORD56-P14 + - ORD58-P11 + Date: + - Wed, 04 Feb 2026 21:02:21 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 812145af-a9a3-49fd-84c9-d39181d7718a + X-Bt-Internal-Trace-Id: + - 6983b3dd00000000399a5d0fdf5278f4 + X-Amz-Apigw-Id: + - YRkKmEllIAMEIvQ= + Vary: + - Origin + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-6983b3dd-6eff774501558c744afec34a;Parent=17a43cf95b7e41bd;Sampled=0;Lineage=1:fc3b4ff1:0 + X-Bt-Cursor: + - aYOz3CfMAAA + Via: + - 1.1 c41d7e84814c40081afb134b22aff29a.cloudfront.net (CloudFront), 1.1 30b00cb975432e84bcc7b58f21d34a50.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - a2qxGlEeDaCngbwkkGB-Dl7b3U5__zSCKgPC1AHXSUQLRMwIVVSBNQ== + body: + encoding: ASCII-8BIT + string: | + {"_pagination_key":"p07603118354073387009","_xact_id":"1000196606632142796","audit_data":[{"_xact_id":"1000196606632142796","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-02-04T21:02:20.370Z","dataset_id":"e3cb8850-b409-4a34-a12c-89f93e5331c3","expected":"LAZY-2","facets":null,"id":"734e8320-175f-4240-a2f8-26c42b56e8ed","input":"lazy-2","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"950e3708-6c09-479a-a7ca-d1d1dbd07e0a","span_id":"950e3708-6c09-479a-a7ca-d1d1dbd07e0a","tags":null} + {"_pagination_key":"p07603118354073387008","_xact_id":"1000196606632142796","audit_data":[{"_xact_id":"1000196606632142796","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-02-04T21:02:20.370Z","dataset_id":"e3cb8850-b409-4a34-a12c-89f93e5331c3","expected":"LAZY-1","facets":null,"id":"3c537c61-7129-496a-b304-2e6a62dfc45b","input":"lazy-1","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"f2d45fdc-5cb5-4d1d-8ec5-fa102acacdc4","span_id":"f2d45fdc-5cb5-4d1d-8ec5-fa102acacdc4","tags":null} + recorded_at: Wed, 04 Feb 2026 21:02:21 GMT +recorded_with: VCR 6.4.0 diff --git a/test/fixtures/vcr_cassettes/dataset/fetch_all.yml b/test/fixtures/vcr_cassettes/dataset/fetch_all.yml new file mode 100644 index 0000000..73be39a --- /dev/null +++ b/test/fixtures/vcr_cassettes/dataset/fetch_all.yml @@ -0,0 +1,342 @@ +--- +http_interactions: +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '395' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZDVjMTg3NGQtMDYxZS00MmNhLTlmMTMtZTI2YjM2NzVjMjlh'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 04 Feb 2026 21:02:21 GMT + Etag: + - '"12n7ok4b5phaz"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - ZDVjMTg3NGQtMDYxZS00MmNhLTlmMTMtZTI2YjM2NzVjMjlh + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - cle1::iad1::tcvfp-1770238941191-57b52ca4192e + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Wed, 04 Feb 2026 21:02:21 GMT +- request: + method: post + uri: https://www.braintrust.dev/api/dataset/register + body: + encoding: UTF-8 + string: '{"dataset_name":"test-ruby-sdk-dataset-fetch","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_name":"ruby-sdk-test"}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '615' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZTZjZWI2NjYtMzNkZS00NDQ1LTgwMzQtMjQ5NTUyZjViZDE4'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 04 Feb 2026 21:02:21 GMT + Etag: + - '"6ox8ebhlcyh3"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/dataset/register" + X-Nonce: + - ZTZjZWI2NjYtMzNkZS00NDQ1LTgwMzQtMjQ5NTUyZjViZDE4 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - cle1::iad1::5d7sn-1770238941380-106b01f88323 + body: + encoding: UTF-8 + string: '{"project":{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","description":null,"created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null},"dataset":{"id":"a6c37ef0-1722-40d5-8bfb-19bea7883aea","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-fetch","description":null,"created":"2026-02-04T21:02:21.444Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"url_slug":"test-ruby-sdk-dataset-fetch"},"found_existing":false}' + recorded_at: Wed, 04 Feb 2026 21:02:21 GMT +- request: + method: post + uri: https://staging-api.braintrust.dev/v1/dataset/a6c37ef0-1722-40d5-8bfb-19bea7883aea/insert + body: + encoding: UTF-8 + string: '{"events":[{"input":"fetch-test","expected":"FETCH-TEST"}]}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '52' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - ORD56-P14 + - ORD58-P11 + Date: + - Wed, 04 Feb 2026 21:02:21 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 7aad9745-b75c-4d4c-b605-13678d423f9c + X-Bt-Internal-Trace-Id: + - 6983b3dd000000007521b749869bba4b + X-Amz-Apigw-Id: + - YRkKrEusoAMEVfA= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"34-VSm8tDVY1XsOwsbproLq4jbikl4" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-6983b3dd-00054db8551f83304b4de7fd;Parent=3a622592a4f6f2f5;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 fd1d7bb9e75d267e13b5531a5444e9c0.cloudfront.net (CloudFront), 1.1 a4f9034f040b2c72126eaff1ca10fb64.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - hBCKowIecxthcMu9GhvLFLkcwZ9S33HkL4ZlvE9hNkfjMpEaXVeQ1w== + body: + encoding: ASCII-8BIT + string: '{"row_ids":["5c987495-5126-4d26-91ba-44bd0900d979"]}' + recorded_at: Wed, 04 Feb 2026 21:02:21 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/v1/dataset?dataset_name=test-ruby-sdk-dataset-fetch&project_name=ruby-sdk-test + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '328' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - ORD56-P14 + - ORD58-P11 + Date: + - Wed, 04 Feb 2026 21:02:22 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 46d181af-b9b3-42ea-a643-b0dd83bb4e82 + X-Bt-Internal-Trace-Id: + - 6983b3de00000000595fbce15bd4588b + X-Amz-Apigw-Id: + - YRkKvGFkIAMELdw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"148-gd16NfVM8NL/YdTmdxzjZS4wpsM" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-6983b3dd-0ece41a81ed43d1245a91671;Parent=0530475da53fe90b;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 fd1d7bb9e75d267e13b5531a5444e9c0.cloudfront.net (CloudFront), 1.1 ad00f514084d54a9d0f9415483c3c482.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - rzmbEGkqPvN3ks8H9ut_RRv28wUwf8PkKcNyk3KHIkg10pGeWDgbwQ== + body: + encoding: ASCII-8BIT + string: '{"objects":[{"id":"a6c37ef0-1722-40d5-8bfb-19bea7883aea","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-fetch","description":null,"created":"2026-02-04T21:02:21.444Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"url_slug":"test-ruby-sdk-dataset-fetch"}]}' + recorded_at: Wed, 04 Feb 2026 21:02:22 GMT +- request: + method: post + uri: https://staging-api.braintrust.dev/btql + body: + encoding: UTF-8 + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"a6c37ef0-1722-40d5-8bfb-19bea7883aea"}]},"select":[{"op":"star"}],"limit":1},"fmt":"jsonl"}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Content-Length: + - '632' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - ORD56-P14 + - ORD58-P11 + Date: + - Wed, 04 Feb 2026 21:02:22 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 53960308-ff68-4a05-bead-6f6bf97fede4 + X-Bt-Internal-Trace-Id: + - 6983b3de000000003e1e92d474e42938 + X-Amz-Apigw-Id: + - YRkKyES2oAMEDeg= + Vary: + - Origin + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-6983b3de-6344d8616cd1c6893ff78e59;Parent=581fa2d0be457743;Sampled=0;Lineage=1:fc3b4ff1:0 + X-Bt-Cursor: + - aYOz3TAHAAA + Via: + - 1.1 bf89839dd141364aec15e919e53bb54a.cloudfront.net (CloudFront), 1.1 180a4e1c2aa6a4d95dc2a1b2b749e54a.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - J6NnnM4fWEESiijIGhWHZWXHeYe4kiRusqMh0YcMN1rJUuo4aQOu-w== + body: + encoding: ASCII-8BIT + string: '{"_pagination_key":"p07603118358506438656","_xact_id":"1000196606632210439","audit_data":[{"_xact_id":"1000196606632210439","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-02-04T21:02:21.580Z","dataset_id":"a6c37ef0-1722-40d5-8bfb-19bea7883aea","expected":"FETCH-TEST","facets":null,"id":"5c987495-5126-4d26-91ba-44bd0900d979","input":"fetch-test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"23316edc-60a5-4375-af53-45219361db96","span_id":"23316edc-60a5-4375-af53-45219361db96","tags":null} + + ' + recorded_at: Wed, 04 Feb 2026 21:02:22 GMT +recorded_with: VCR 6.4.0 diff --git a/test/fixtures/vcr_cassettes/eval/dataset_hash_id.yml b/test/fixtures/vcr_cassettes/eval/dataset_hash_id.yml index 55950f0..7b0326a 100644 --- a/test/fixtures/vcr_cassettes/eval/dataset_hash_id.yml +++ b/test/fixtures/vcr_cassettes/eval/dataset_hash_id.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-ZDU3OWQwMTAtYzQzZC00N2YxLWFjNzctNDkxMzkyYmM5YTEx'' *.js.stripe.com + ''nonce-ZmFiY2NjYzAtMGRlZS00MmI5LWJlM2MtYjUwNTliZGM1YmVi'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:31 GMT + - Wed, 04 Feb 2026 21:00:10 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,21 +71,21 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - ZDU3OWQwMTAtYzQzZC00N2YxLWFjNzctNDkxMzkyYmM5YTEx + - ZmFiY2NjYzAtMGRlZS00MmI5LWJlM2MtYjUwNTliZGM1YmVi X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::vgjfx-1761319769584-2ffc8b24e6a3 + - cle1::iad1::q45cl-1770238810459-b9631406fadc body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:31 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Wed, 04 Feb 2026 21:00:10 GMT - request: method: post uri: https://www.braintrust.dev/api/dataset/register body: encoding: UTF-8 - string: '{"dataset_name":"test-ruby-sdk-dataset-id","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","project_name":"ruby-sdk-test"}' + string: '{"dataset_name":"test-ruby-sdk-dataset-id","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_name":"ruby-sdk-test"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -106,24 +107,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '551' + - '613' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-ZmZmMDdjMjgtNmNkZS00MTM4LTg0MDAtYmNmNGVmZjU2NjQ0'' *.js.stripe.com + ''nonce-NGIxZDM3ZTItYTZmYi00NWIwLWE3M2MtM2NmZTU4Yzk1Yzcx'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:32 GMT + - Wed, 04 Feb 2026 21:00:11 GMT Etag: - - '"phszpfole9fb"' + - '"niv3d9z2h3h1"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -142,18 +144,18 @@ http_interactions: X-Matched-Path: - "/api/dataset/register" X-Nonce: - - ZmZmMDdjMjgtNmNkZS00MTM4LTg0MDAtYmNmNGVmZjU2NjQ0 + - NGIxZDM3ZTItYTZmYi00NWIwLWE3M2MtM2NmZTU4Yzk1Yzcx X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::p9gw7-1761319771936-769b9ea74a18 + - cle1::iad1::ssx6j-1770238810813-a80db886dd0f body: encoding: UTF-8 - string: '{"project":{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null},"dataset":{"id":"f6040259-2f57-463c-ad11-5f0976480ae0","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-dataset-id","description":null,"created":"2025-10-24T14:47:37.853Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null},"found_existing":true}' - recorded_at: Fri, 24 Oct 2025 15:29:32 GMT + string: '{"project":{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","description":null,"created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null},"dataset":{"id":"02c8546e-38f8-46c1-8b09-df9c08854341","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-id","description":null,"created":"2025-12-11T18:38:01.718Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"url_slug":"test-ruby-sdk-dataset-id-675d"},"found_existing":true}' + recorded_at: Wed, 04 Feb 2026 21:00:11 GMT - request: method: post - uri: https://api.braintrust.dev/v1/dataset/f6040259-2f57-463c-ad11-5f0976480ae0/insert + uri: https://staging-api.braintrust.dev/v1/dataset/02c8546e-38f8-46c1-8b09-df9c08854341/insert body: encoding: UTF-8 string: '{"events":[{"input":"test","expected":"TEST"}]}' @@ -165,7 +167,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -182,43 +184,43 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - ORD56-P14 + - ORD58-P11 Date: - - Fri, 24 Oct 2025 15:29:32 GMT + - Wed, 04 Feb 2026 21:00:11 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 3800ff4e-76c3-4db9-ba76-b1dfac848b52 + - 5476c481-f16b-475b-a308-43281f62e47b X-Bt-Internal-Trace-Id: - - 68fb9b5c000000004e00408c3245ae99 + - 6983b35b0000000069f8356c1c66e8de X-Amz-Apigw-Id: - - S9U2eEN_IAMEURw= + - YRj2UGRjIAMESXA= Vary: - Origin, Accept-Encoding Etag: - - W/"34-kpTdz8lS672FvIsIVFHZ8Z75loI" + - W/"34-P9JaNvsANzEZIKsHgSMenLDGVKM" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5c-001111f77272d9132d153b73;Parent=46b22aeb9d81f6e3;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6983b35b-4d4ce79c476d19ca77b504c5;Parent=2effe52defc6ca8b;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 f8debc28b6c73eb3dc7540e2ac2f0e18.cloudfront.net (CloudFront), 1.1 f25b89e7ef738cb8bb7e28e041d8fe54.cloudfront.net + - 1.1 3ec1da3c3a305aac0aa854f904917156.cloudfront.net (CloudFront), 1.1 f7da53b1e8d211ec11d888343978cd1c.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - QEXg_Jf2pT3XKOlRi2HB04ROe093kcGpWh_Fbaitn7ngDXjZvFR_Jg== + - DuV2ndRRgJOiXP2xMoOb2e3JCJRBaxqvoMDYKHfR8YbVCyvm8a4TfQ== body: encoding: ASCII-8BIT - string: '{"row_ids":["5dcd0c01-13dd-4bd6-842f-5478c963d4d2"]}' - recorded_at: Fri, 24 Oct 2025 15:29:32 GMT + string: '{"row_ids":["968885fb-cbdc-4c37-ba5d-67704eaae05d"]}' + recorded_at: Wed, 04 Feb 2026 21:00:11 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"f6040259-2f57-463c-ad11-5f0976480ae0"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"02c8546e-38f8-46c1-8b09-df9c08854341"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -227,7 +229,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -240,54 +242,54 @@ http_interactions: Content-Type: - application/json Content-Length: - - '3185' + - '4340' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - ORD56-P14 + - ORD58-P11 Date: - - Fri, 24 Oct 2025 15:29:32 GMT + - Wed, 04 Feb 2026 21:00:12 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - a6e021fa-d559-4505-ba77-0ffbc96bcff5 + - 2ed50e04-1ef5-4ba8-b101-19a49f308b64 X-Bt-Internal-Trace-Id: - - 68fb9b5c0000000052c56f9dddfb5706 + - 6983b35b0000000079b860657f0b097c X-Amz-Apigw-Id: - - S9U2hFzOoAMEHzA= + - YRj2YEMCoAMEpyg= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5c-0cbcfbc5668735d17956d33b;Parent=668dcfa397621f76;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6983b35b-3e6e47452c50ce6e4aaa31eb;Parent=19912ce4e0bca580;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Cursor: - - aPuRis2rAAA + - aTsPiuPtAAA Via: - - 1.1 f5527f719bbc0d2932043daaeff80252.cloudfront.net (CloudFront), 1.1 d50d90bbddca57e02d6288d86c88470a.cloudfront.net + - 1.1 3fac42b1ad18d829a2dc59e237bdaa3c.cloudfront.net (CloudFront), 1.1 0346ce001870d6b715406f7d48c2a0ee.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - o7_aLL_9GHCnKNur2O73gmvcChz1LMldLQwPxhsXQ4HgZNr0wr9Jig== + - 4jxMMJl7-15Z1TlAKuY_-65KsZm7aXNYEqN8wA84eGpxFEAm-_gGXg== body: encoding: ASCII-8BIT string: | - {"_pagination_key":"p07564810819080880128","_xact_id":"1000196022105546841","created":"2025-10-24T15:29:32.318Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"5dcd0c01-13dd-4bd6-842f-5478c963d4d2","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"cbe220a1-4f3c-4be0-a910-26bf87f11f59","span_id":"cbe220a1-4f3c-4be0-a910-26bf87f11f59","tags":null} - {"_pagination_key":"p07564802288482189312","_xact_id":"1000196021975380235","created":"2025-10-24T14:56:25.536Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"9346985b-ac19-47e4-81cf-8aa0f0f0176a","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"3816c372-b36e-4e05-88a6-5e86254050bb","span_id":"3816c372-b36e-4e05-88a6-5e86254050bb","tags":null} - {"_pagination_key":"p07564801958911475712","_xact_id":"1000196021970351385","created":"2025-10-24T14:55:09.030Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"91594e1e-bca7-4d82-a9a9-2d294aea954d","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"102467dc-10f4-4a17-94ba-8358587bfe80","span_id":"102467dc-10f4-4a17-94ba-8358587bfe80","tags":null} - {"_pagination_key":"p07564801559978508288","_xact_id":"1000196021964264151","created":"2025-10-24T14:53:36.427Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"210f2cb4-bf0a-4ae3-b7fd-53b70a5bdc4b","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"83d7412d-8dd2-41c7-9895-8a5719c86e75","span_id":"83d7412d-8dd2-41c7-9895-8a5719c86e75","tags":null} - {"_pagination_key":"p07564800887563288576","_xact_id":"1000196021954003909","created":"2025-10-24T14:50:59.550Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"e9c2225b-59ba-4796-98dc-61219ab4828c","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"9b630e57-d51b-4cf0-b316-19c8fb089013","span_id":"9b630e57-d51b-4cf0-b316-19c8fb089013","tags":null} - {"_pagination_key":"p07564800709715558400","_xact_id":"1000196021951290168","created":"2025-10-24T14:50:17.976Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"e5527e26-fe2f-4517-b8ca-0fd138d980be","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"53b1f92e-c259-4559-9e56-e31c194f922c","span_id":"53b1f92e-c259-4559-9e56-e31c194f922c","tags":null} - {"_pagination_key":"p07564800024440930304","_xact_id":"1000196021940833707","created":"2025-10-24T14:47:37.921Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"32e3891f-69fa-4d89-be8b-b9c691f5fe2f","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"e5e96bf1-209e-4d96-bb90-e39dd5bab4f0","span_id":"e5e96bf1-209e-4d96-bb90-e39dd5bab4f0","tags":null} - recorded_at: Fri, 24 Oct 2025 15:29:32 GMT + {"_pagination_key":"p07603117802490626048","_xact_id":"1000196606623726311","audit_data":[{"_xact_id":"1000196606623726311","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-02-04T21:00:11.263Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","facets":null,"id":"968885fb-cbdc-4c37-ba5d-67704eaae05d","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"663e2ffd-fa36-40ed-bdc9-51e8de705162","span_id":"663e2ffd-fa36-40ed-bdc9-51e8de705162","tags":null} + {"_pagination_key":"p07603072546923347968","_xact_id":"1000196605933181156","audit_data":[{"_xact_id":"1000196605933181156","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-02-04T18:04:34.272Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","facets":null,"id":"1f7e18a5-fdeb-4304-93c8-83473c45c030","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"c1cc9ca1-b8c5-4223-8d24-053e79cce175","span_id":"c1cc9ca1-b8c5-4223-8d24-053e79cce175","tags":null} + {"_pagination_key":"p07600490767317794816","_xact_id":"1000196566538350749","audit_data":[{"_xact_id":"1000196566538350749","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-01-28T19:05:57.752Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","facets":null,"id":"3c35aee9-ebab-4ba0-9b24-db2d68a11e94","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"97b0b448-9f77-4fc3-a398-8fcfc5bd557d","span_id":"97b0b448-9f77-4fc3-a398-8fcfc5bd557d","tags":null} + {"_pagination_key":"p07582675098489389056","_xact_id":"1000196294692818089","audit_data":[{"_xact_id":"1000196294692818089","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2025-12-11T18:52:03.010Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","facets":null,"id":"eef9aa47-378d-4752-9f8c-72f8021ee763","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"3be90d25-5d85-4eaf-b573-c4c41a9c0f8a","span_id":"3be90d25-5d85-4eaf-b573-c4c41a9c0f8a","tags":null} + {"_pagination_key":"p07582672561744052224","_xact_id":"1000196294654110427","audit_data":[{"_xact_id":"1000196294654110427","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2025-12-11T18:42:13.699Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","facets":null,"id":"2787844c-196e-4446-b5a7-1d255bf2cc15","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"07f805a3-e68b-4990-a5a9-367ea21d3b28","span_id":"07f805a3-e68b-4990-a5a9-367ea21d3b28","tags":null} + {"_pagination_key":"p07582671935010111488","_xact_id":"1000196294644547226","audit_data":[{"_xact_id":"1000196294644547226","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2025-12-11T18:39:47.354Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","facets":null,"id":"77a419b5-c01a-45b5-b224-9a2843c1c3fa","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"a62f2dd8-6e60-46bc-904d-be55e49c18bf","span_id":"a62f2dd8-6e60-46bc-904d-be55e49c18bf","tags":null} + {"_pagination_key":"p07582671486812225536","_xact_id":"1000196294637708269","audit_data":[{"_xact_id":"1000196294637708269","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2025-12-11T18:38:01.811Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","facets":null,"id":"a2832a5b-129e-454c-8d0d-6d2483948616","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"af66dac9-542f-4ea8-9b52-922234f4d3f0","span_id":"af66dac9-542f-4ea8-9b52-922234f4d3f0","tags":null} + recorded_at: Wed, 04 Feb 2026 21:00:12 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"f6040259-2f57-463c-ad11-5f0976480ae0"}]},"select":[{"op":"star"}],"limit":1000,"cursor":"aPuRis2rAAA"},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"02c8546e-38f8-46c1-8b09-df9c08854341"}]},"select":[{"op":"star"}],"limit":1000,"cursor":"aTsPiuPtAAA"},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -296,7 +298,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -313,38 +315,38 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - ORD56-P14 + - ORD58-P11 Date: - - Fri, 24 Oct 2025 15:29:33 GMT + - Wed, 04 Feb 2026 21:00:12 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 17a8f9b4-31e9-483b-a158-de887df9f9fc + - 28998d8f-f492-4268-87e2-13b5588c2a4c X-Bt-Internal-Trace-Id: - - 68fb9b5c0000000032a041840c2be844 + - 6983b35c00000000508eee2a97ac93b2 X-Amz-Apigw-Id: - - S9U2lEkkIAMEiqg= + - YRj2dH3yIAMEvKg= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5c-46cca62608748eca550bdd58;Parent=4993f4968cf0f988;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6983b35c-105d06863c5507405a96f8cb;Parent=055c250f30cb6255;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 bf8d7cb6fca5d51158e1109ca40fe242.cloudfront.net (CloudFront), 1.1 8cdf4e2d4f4070992665477c4dbca0c0.cloudfront.net + - 1.1 5eb83646e0a82bf058585a903394df46.cloudfront.net (CloudFront), 1.1 bcfa57e974ea0a8b42c18a98782a28ec.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - lvvQgSq6gldMnrfwR4nOxSSEk6x2Aj48ljAzzhRH7pDYgrs3huxboA== + - kIlHU3C94XDrc0MG0Cw3LqpGI_9bjyODFUFWzbrARiOjukxfB57usw== body: encoding: ASCII-8BIT string: '' - recorded_at: Fri, 24 Oct 2025 15:29:33 GMT + recorded_at: Wed, 04 Feb 2026 21:00:12 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -356,7 +358,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -369,49 +371,49 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '236' + - '255' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - ORD56-P14 + - ORD58-P11 Date: - - Fri, 24 Oct 2025 15:29:33 GMT + - Wed, 04 Feb 2026 21:00:12 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 607955de-d262-443d-a6d3-bf72dd3c11a0 + - f9a84f47-ba5f-4e78-b5f3-1c6da6a286a5 X-Bt-Internal-Trace-Id: - - 68fb9b5d000000007de0590e1ad67207 + - 6983b35c00000000741077af41cb2e42 X-Amz-Apigw-Id: - - S9U2pGd1oAMEU7A= + - YRj2eFuaIAMEStA= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ff-6vJBGKzQtkVK0GxiGBpsJc6l6u8" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5d-6e4c1c777d42427d634ccead;Parent=528fc2cd1cab4cda;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6983b35c-7305fac9220a8b8b76bac738;Parent=6fa6fc850b821cd8;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 bd3e3884ce6fe1fd36336541cce9ec7e.cloudfront.net + - 1.1 5eb83646e0a82bf058585a903394df46.cloudfront.net (CloudFront), 1.1 a1df6d6150d749ef6922ff69e92bb5b6.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - Fk3lrm3cdhGym9QkfoNZSUkK1c_9bTBCxrBnqaX8OMatSGuoeuy30A== + - Rq4tHoNmcY1vteSC6viK1C9pubjIHMtKXFzV4EE7tgmlEJkMZpfgwA== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:33 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","description":null,"created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Wed, 04 Feb 2026 21:00:12 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-id","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-id","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -420,7 +422,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -433,39 +435,39 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '391' + - '393' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - ORD56-P14 + - ORD58-P11 Date: - - Fri, 24 Oct 2025 15:29:33 GMT + - Wed, 04 Feb 2026 21:00:12 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 4b6699da-8ccc-4559-83a8-b78adff6c860 + - 258822ff-ff30-473c-9782-be824c7327cc X-Bt-Internal-Trace-Id: - - 68fb9b5d000000006c2affe40afd57a8 + - 6983b35c000000007496449fbe311b2f X-Amz-Apigw-Id: - - S9U2rEHEoAMEjeg= + - YRj2hEDuIAMEMow= Vary: - Origin, Accept-Encoding Etag: - - W/"187-vzidsy1hbvmgrjVQ0RAXiEKH2Rc" + - W/"189-qXRZojY0TirnXlmRnFnXtcP3Mv0" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5d-27b77f47569a33c617bb6986;Parent=453497dffe353ea7;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-6983b35c-70479649494d1e2629a8f6b9;Parent=6afa1bd4f9a3f510;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 8a9cdb228e33f8d52a4b42c56ca26590.cloudfront.net (CloudFront), 1.1 76d4de5b65bdf749a3f97445d1b9f4d2.cloudfront.net + - 1.1 d90c6f2271c89398e6144cc762852e94.cloudfront.net (CloudFront), 1.1 7afb03828af3b7a61fd0b35f32c9822a.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - RTikKucW-hlXS7I0JEkE89suRnArte11DMJxtX6vABjgKyXeQB7vVQ== + - B1GtfyzYXl8-3_BL4qVOG4bzgH2sbUzzNVgufwLwJQj8uQ2vnzeXMA== body: encoding: ASCII-8BIT - string: '{"id":"6e338119-5266-4979-abcf-9930e165bd3b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-id-975ec196","description":null,"created":"2025-10-24T15:29:33.607Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:33 GMT -recorded_with: VCR 6.3.1 + string: '{"id":"352c8b58-caa6-41b3-8b14-fe4242f7cfa9","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-id-46a8c2b5","description":null,"created":"2026-02-04T21:00:12.730Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Wed, 04 Feb 2026 21:00:12 GMT +recorded_with: VCR 6.4.0 diff --git a/test/support/braintrust_helper.rb b/test/support/braintrust_helper.rb index b5ff6c0..43232a4 100644 --- a/test/support/braintrust_helper.rb +++ b/test/support/braintrust_helper.rb @@ -64,8 +64,8 @@ def get_integration_test_state(**options) Braintrust.init(set_global: false, blocking_login: true, **options) end - # Creates an API client for integration tests (without polluting global state) - # This is the preferred way to get an API client for tests. + # Creates an API for integration tests + # This is the preferred way to get an API for tests. # @param options [Hash] Options to pass to get_integration_test_state # @return [Braintrust::API] def get_integration_test_api(**options) @@ -74,8 +74,9 @@ def get_integration_test_api(**options) end # Helper to run eval internally without API calls for testing + # @param api [API] Braintrust API client def run_test_eval(experiment_id:, experiment_name:, project_id:, project_name:, - cases:, task:, scorers:, state:, parallelism: 1, tracer_provider: nil) + cases:, task:, scorers:, api:, parallelism: 1, tracer_provider: nil) runner = Braintrust::Eval::Runner.new( experiment_id: experiment_id, experiment_name: experiment_name, @@ -83,7 +84,7 @@ def run_test_eval(experiment_id:, experiment_name:, project_id:, project_name:, project_name: project_name, task: task, scorers: scorers, - state: state, + api: api, tracer_provider: tracer_provider ) runner.run(cases, parallelism: parallelism) diff --git a/test/support/tracing_helper.rb b/test/support/tracing_helper.rb index 3a0ace7..4be57b4 100644 --- a/test/support/tracing_helper.rb +++ b/test/support/tracing_helper.rb @@ -37,6 +37,12 @@ def initialize(tracer_provider, exporter, state) @state = state end + # Get API client for the test state + # @return [Braintrust::API] + def api + @api ||= Braintrust::API.new(state: @state) + end + # Get a tracer from the provider # @param name [String] tracer name (default: "test") # @return [OpenTelemetry::Trace::Tracer]