From 79b1001aff4e52344e53d5a964ca25d1b24c8afb Mon Sep 17 00:00:00 2001 From: Ismael G Marin C Date: Tue, 30 Jun 2026 14:07:04 -0600 Subject: [PATCH 1/2] feat: per-run token and cost accounting (#40) --- .../agent/react_agent/loop_runner.rb | 53 +++++++++-- lib/skill_bench/agent/react_agent/step.rb | 8 +- lib/skill_bench/output_formatter.rb | 14 +++ .../services/agent_spawner_service.rb | 10 ++- lib/skill_bench/services/cost_calculator.rb | 89 +++++++++++++++++++ lib/skill_bench/services/json_formatter.rb | 22 ++++- lib/skill_bench/services/runner_service.rb | 62 +++++++++++++ test/agent_eval/output_formatter_test.rb | 29 ++++++ .../services/agent_spawner_service_test.rb | 46 ++++++++++ .../services/runner_service_test.rb | 27 ++++++ .../evaluator/react_agent/loop_runner_test.rb | 40 +++++++++ test/evaluator/react_agent/step_test.rb | 46 ++++++++++ test/services/cost_calculator_test.rb | 74 +++++++++++++++ test/services/json_formatter_test.rb | 19 ++++ 14 files changed, 525 insertions(+), 14 deletions(-) create mode 100644 lib/skill_bench/services/cost_calculator.rb create mode 100644 test/services/cost_calculator_test.rb diff --git a/lib/skill_bench/agent/react_agent/loop_runner.rb b/lib/skill_bench/agent/react_agent/loop_runner.rb index 8b6a71c..788eb0f 100644 --- a/lib/skill_bench/agent/react_agent/loop_runner.rb +++ b/lib/skill_bench/agent/react_agent/loop_runner.rb @@ -16,6 +16,7 @@ class LoopRunner def self.call(initial_prompt, max_iterations, config) messages = [{ role: 'user', content: initial_prompt }] iterations_log = [] + total_usage = empty_usage step_count = 0 while step_count < max_iterations @@ -24,24 +25,27 @@ def self.call(initial_prompt, max_iterations, config) step_result = Step.call(messages, config) iteration = step_result[:iteration] iterations_log << attach_step_number(iteration, step_count) if iteration + total_usage = add_usage(total_usage, step_result[:usage]) unless step_result[:continue] final_result = step_result[:result] || { success: false, response: { error: { message: 'Step returned no result' } } } - return merge_iterations(final_result, iterations_log) + return finalize(final_result, iterations_log, total_usage) end messages = step_result[:messages] end - merge_iterations( + finalize( { success: false, response: { error: { message: Agent::ReactAgent::MAX_ITERATIONS_REACHED } } }, - iterations_log + iterations_log, + total_usage ) rescue StandardError => e SkillBench::ErrorLogger.log_error(e, 'ReactAgent Error') - merge_iterations( + finalize( { success: false, response: { error: { message: e.message } } }, - iterations_log + iterations_log, + total_usage ) end @@ -54,14 +58,45 @@ def self.attach_step_number(iteration, step_count) iteration.merge(step_number: step_count) end - # Merges the collected iterations into the result response. + # Merges the collected iterations and accumulated usage into the response. # # @param result [Hash] The final result hash from the loop. # @param iterations_log [Array] Collected iteration metadata. - # @return [Hash] The result with :iterations injected into :response. - def self.merge_iterations(result, iterations_log) + # @param total_usage [Hash] Summed token usage across all iterations. + # @return [Hash] The result with :iterations and :usage injected into :response. + def self.finalize(result, iterations_log, total_usage) response = result[:response] || {} - result.merge(response: response.merge(iterations: iterations_log)) + result.merge(response: response.merge(iterations: iterations_log, usage: total_usage)) + end + + # A zeroed token-usage accumulator. + # + # @return [Hash] Usage hash with prompt/completion/total token counts set to zero. + def self.empty_usage + { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 } + end + + # Adds a single step's usage onto a running total. + # + # @param total [Hash] The running usage total. + # @param usage [Hash, nil] A step's usage hash (may be nil or empty). + # @return [Hash] A new summed usage hash. + def self.add_usage(total, usage) + usage ||= {} + { + prompt_tokens: total[:prompt_tokens] + token_count(usage, :prompt_tokens), + completion_tokens: total[:completion_tokens] + token_count(usage, :completion_tokens), + total_tokens: total[:total_tokens] + token_count(usage, :total_tokens) + } + end + + # Reads a token count from a usage hash, tolerating string keys. + # + # @param usage [Hash] The usage hash. + # @param key [Symbol] The usage key (e.g. :prompt_tokens). + # @return [Integer] The token count, or zero when absent. + def self.token_count(usage, key) + (usage[key] || usage[key.to_s] || 0).to_i end end end diff --git a/lib/skill_bench/agent/react_agent/step.rb b/lib/skill_bench/agent/react_agent/step.rb index b42d51d..e9713f8 100644 --- a/lib/skill_bench/agent/react_agent/step.rb +++ b/lib/skill_bench/agent/react_agent/step.rb @@ -12,7 +12,8 @@ class Step # # @param messages [Array] The conversation history. # @param config [Hash] Configuration for this step (client params, system prompt, working dir). - # @return [Hash] Step outcome containing :continue (boolean), :result (hash, if finished), and :messages. + # @return [Hash] Step outcome containing :continue (boolean), :result (hash, if finished), + # :usage (token usage for this step), and :messages. def self.call(messages, config) messages = messages.dup client_result = Client.call( @@ -21,12 +22,14 @@ def self.call(messages, config) tools: Tools.definitions, **config[:client_params] ) + usage = client_result[:usage] || {} unless client_result[:success] error_msg = client_result.dig(:response, :error, :message) || 'Unknown error' return { continue: false, result: client_result, + usage: usage, iteration: build_iteration(thought: '', tools_used: [], observation_summary: error_msg) } end @@ -36,6 +39,7 @@ def self.call(messages, config) return { continue: false, result: { success: false, response: { error: { message: 'Empty response from LLM' } } }, + usage: usage, iteration: build_iteration(thought: '', tools_used: [], observation_summary: 'Empty response from LLM') } end @@ -51,6 +55,7 @@ def self.call(messages, config) return { continue: false, result: { success: true, response: { content: content } }, + usage: usage, iteration: build_iteration(thought: thought, tools_used: [], observation_summary: '') } end @@ -69,6 +74,7 @@ def self.call(messages, config) { continue: true, messages: messages, + usage: usage, iteration: build_iteration(thought: thought, tools_used: tools_used, observation_summary: observation_summary) } end diff --git a/lib/skill_bench/output_formatter.rb b/lib/skill_bench/output_formatter.rb index 98140e1..55f4cdf 100644 --- a/lib/skill_bench/output_formatter.rb +++ b/lib/skill_bench/output_formatter.rb @@ -135,6 +135,7 @@ def self.format_delta_report(result, report) " Eval: #{result[:eval_name] || ''}", " Skill: #{result[:skill_name] || ''}", " Provider: #{result[:provider_name] || ''}", + build_usage_line(result), ('═' * 55), '' ] @@ -152,6 +153,19 @@ def self.format_delta_report(result, report) end private_class_method :format_delta_report + # Builds the token/cost summary line for the report header. + # + # @param result [Hash] Eval result envelope; reads :tokens and :cost. + # @return [String] A formatted "Tokens / Est. Cost" line. + def self.build_usage_line(result) + tokens = result[:tokens] || {} + total = tokens[:total_tokens] || tokens['total_tokens'] || 0 + cost = result[:cost] + cost_label = cost ? Kernel.format('$%.4f', cost) : '—' + " Tokens: #{total} | Est. Cost: #{cost_label}" + end + private_class_method :build_usage_line + # Builds iteration timeline lines from the result response. # # @param result [Hash] Eval result envelope. diff --git a/lib/skill_bench/services/agent_spawner_service.rb b/lib/skill_bench/services/agent_spawner_service.rb index 47f3702..98bd71f 100644 --- a/lib/skill_bench/services/agent_spawner_service.rb +++ b/lib/skill_bench/services/agent_spawner_service.rb @@ -7,6 +7,9 @@ module SkillBench module Services # Spawns and executes LLM agents for evaluation. class AgentSpawnerService + # Zeroed token usage used when a run produces no usage data (e.g. mock, rescue). + EMPTY_USAGE = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }.freeze + # Spawns the LLM agent with the given system prompt. # # @param evaluation [SkillBench::Models::Eval] The eval being run @@ -33,7 +36,7 @@ def initialize(evaluation, system_prompt, provider, config) # # @return [Hash] Agent response with result, status, runtime, usage, raw_response, iterations def call - return { result: 'mock result', status: :success, iterations: [] } if @provider.name == 'mock' + return { result: 'mock result', status: :success, iterations: [], usage: EMPTY_USAGE } if @provider.name == 'mock' client_params = build_client_params max_iterations = @config&.[](:max_iterations) || @config&.[]('max_iterations') || 25 @@ -63,6 +66,7 @@ def run_agent(client_params, max_iterations) final_answer = agent_result.dig(:response, :content) || '' diff = Execution::Sandbox.capture_diff(sandbox.path) iterations = agent_result.dig(:response, :iterations) || [] + usage = agent_result.dig(:response, :usage) || EMPTY_USAGE output = [final_answer, diff].reject(&:empty?).join("\n\n") @@ -70,7 +74,7 @@ def run_agent(client_params, max_iterations) result: output, status: status, runtime: @provider.runtime, - usage: {}, + usage: usage, raw_response: agent_result, iterations: iterations } @@ -80,7 +84,7 @@ def run_agent(client_params, max_iterations) result: "Error: #{e.message}", status: :error, runtime: @provider.runtime, - usage: {}, + usage: EMPTY_USAGE, raw_response: { error: e.message, backtrace: e.backtrace }, iterations: [] } diff --git a/lib/skill_bench/services/cost_calculator.rb b/lib/skill_bench/services/cost_calculator.rb new file mode 100644 index 0000000..addf8ac --- /dev/null +++ b/lib/skill_bench/services/cost_calculator.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module SkillBench + module Services + # Estimates the USD cost of an LLM run from token usage and a model name. + # + # Prices are approximate, drawn from public OpenAI/Anthropic pricing pages, + # and expressed in USD per 1,000 tokens. Provider pricing changes over time, + # so treat the result as a rough estimate and extend {PRICES} as needed. + class CostCalculator + # Approximate per-model prices in USD per 1,000 tokens. + # Keyed by a canonical model prefix; longer prefixes win on lookup so that + # dated variants (e.g. "claude-3-5-sonnet-20241022") resolve correctly. + # Source: public OpenAI and Anthropic pricing pages (approximate). + PRICES = { + 'gpt-4o-mini' => { input: 0.00015, output: 0.0006 }, + 'gpt-4o' => { input: 0.005, output: 0.015 }, + 'gpt-4-turbo' => { input: 0.01, output: 0.03 }, + 'gpt-4' => { input: 0.03, output: 0.06 }, + 'gpt-3.5-turbo' => { input: 0.0005, output: 0.0015 }, + 'claude-3-5-sonnet' => { input: 0.003, output: 0.015 }, + 'claude-3-5-haiku' => { input: 0.0008, output: 0.004 }, + 'claude-3-opus' => { input: 0.015, output: 0.075 }, + 'claude-3-sonnet' => { input: 0.003, output: 0.015 }, + 'claude-3-haiku' => { input: 0.00025, output: 0.00125 } + }.freeze + + # Token count that one priced unit of {PRICES} covers. + TOKENS_PER_UNIT = 1000.0 + + # Estimates the USD cost for a run. + # + # @param usage [Hash, nil] Token usage with :prompt_tokens and :completion_tokens. + # @param model [String, nil] The model name (e.g. "gpt-4o"). + # @return [Float, nil] Estimated cost in USD, or nil when the model is unknown. + def self.call(usage:, model:) + new(usage, model).call + end + + # @param usage [Hash, nil] Token usage hash. + # @param model [String, nil] The model name. + def initialize(usage, model) + @usage = usage || {} + @model = model + end + + # Estimates the USD cost for the configured usage and model. + # + # @return [Float, nil] Estimated cost in USD, or nil when the model is unknown. + def call + price = price_for(@model) + return nil unless price + + input_cost = units(:prompt_tokens) * price[:input] + output_cost = units(:completion_tokens) * price[:output] + (input_cost + output_cost).round(6) + end + + private + + # Finds the price entry for a model by longest matching name prefix. + # + # @param model [String, nil] The model name. + # @return [Hash, nil] Price entry with :input and :output, or nil when unknown. + def price_for(model) + key = model.to_s.downcase + return PRICES[key] if PRICES.key?(key) + + PRICES.select { |name, _| key.start_with?(name) }.max_by { |name, _| name.length }&.last + end + + # Converts a usage token count into priced 1K-token units. + # + # @param key [Symbol] The usage key to read. + # @return [Float] The number of priced units. + def units(key) + token_count(key) / TOKENS_PER_UNIT + end + + # Reads a token count from the usage hash, tolerating string keys. + # + # @param key [Symbol] The usage key (e.g. :prompt_tokens). + # @return [Integer] The token count, or zero when absent. + def token_count(key) + (@usage[key] || @usage[key.to_s] || 0).to_i + end + end + end +end diff --git a/lib/skill_bench/services/json_formatter.rb b/lib/skill_bench/services/json_formatter.rb index ad38c6b..99a039e 100644 --- a/lib/skill_bench/services/json_formatter.rb +++ b/lib/skill_bench/services/json_formatter.rb @@ -6,12 +6,32 @@ module SkillBench module Services # Formats evaluation results as JSON. class JsonFormatter + # Zeroed token usage used when a result carries no usage data. + EMPTY_USAGE = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }.freeze + # Format result as JSON. # + # Ensures top-level :tokens and :cost fields are always present (additive; + # existing keys are preserved) so JSON consumers see a stable shape. + # # @param result [Hash] Eval result. # @return [String] JSON-formatted string. def self.format(result) - JSON.pretty_generate(result) + JSON.pretty_generate(with_usage_fields(result)) + end + + # Returns the result augmented with token/cost fields when missing. + # + # @param result [Hash] Eval result (returned unchanged when not a Hash). + # @return [Hash] Result with :tokens and :cost guaranteed present. + def self.with_usage_fields(result) + return result unless result.is_a?(Hash) + + defaults = { + tokens: result[:tokens] || result.dig(:response, :tokens) || EMPTY_USAGE, + cost: result.key?(:cost) ? result[:cost] : result.dig(:response, :cost) + } + defaults.merge(result) end end end diff --git a/lib/skill_bench/services/runner_service.rb b/lib/skill_bench/services/runner_service.rb index f49f4e7..f307cc9 100644 --- a/lib/skill_bench/services/runner_service.rb +++ b/lib/skill_bench/services/runner_service.rb @@ -11,6 +11,7 @@ require_relative 'judge_params_builder' require_relative 'error_response_builder' require_relative 'trend_recorder_service' +require_relative 'cost_calculator' require_relative 'output_formatter' module SkillBench @@ -155,11 +156,16 @@ def evaluate_and_record_trend(context) trend_result = TrendRecorderService.call(result, eval_name, skill_names) return enrich_error_result(trend_result, evaluation, provider) unless trend_result[:success] + tokens = aggregate_usage(context.baseline_output, context.context_output) + cost = CostCalculator.call(usage: tokens, model: agent_model(config, provider)) + { success: true, eval_name: eval_name, skill_name: skill_names.join(', '), provider_name: provider.name, + tokens: tokens, + cost: cost, response: result[:response].merge( trend: trend_result[:trend], baseline_iterations: context.baseline_output[:iterations] || [], @@ -167,6 +173,62 @@ def evaluate_and_record_trend(context) ) } end + + # Sums the token usage of the baseline and context agent runs. + # + # Judge-side usage is not yet threaded through (the judge lives under the + # untouched `clients/` boundary), so this is scoped to agent usage. + # + # @param baseline_output [Hash] The baseline agent output (carries :usage). + # @param context_output [Hash] The context agent output (carries :usage). + # @return [Hash] Combined prompt/completion/total token counts. + def aggregate_usage(baseline_output, context_output) + add_usage( + add_usage(empty_usage, baseline_output[:usage]), + context_output[:usage] + ) + end + + # A zeroed token-usage accumulator. + # + # @return [Hash] Usage hash with prompt/completion/total token counts set to zero. + def empty_usage + { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 } + end + + # Adds one usage hash onto a running total. + # + # @param total [Hash] The running usage total. + # @param usage [Hash, nil] A run's usage hash (may be nil or empty). + # @return [Hash] A new summed usage hash. + def add_usage(total, usage) + usage ||= {} + { + prompt_tokens: total[:prompt_tokens] + token_count(usage, :prompt_tokens), + completion_tokens: total[:completion_tokens] + token_count(usage, :completion_tokens), + total_tokens: total[:total_tokens] + token_count(usage, :total_tokens) + } + end + + # Reads a token count from a usage hash, tolerating string keys. + # + # @param usage [Hash] The usage hash. + # @param key [Symbol] The usage key (e.g. :prompt_tokens). + # @return [Integer] The token count, or zero when absent. + def token_count(usage, key) + (usage[key] || usage[key.to_s] || 0).to_i + end + + # Resolves the model name used for pricing from config, falling back to the provider LLM. + # + # @param config [Hash, nil] Provider config. + # @param provider [Object] The resolved provider. + # @return [String] The model name (e.g. "gpt-4o"). + def agent_model(config, provider) + return provider.llm unless config.is_a?(Hash) + + config[:model] || config['model'] || provider.llm + end end end end diff --git a/test/agent_eval/output_formatter_test.rb b/test/agent_eval/output_formatter_test.rb index 5a2b292..50ac71e 100644 --- a/test/agent_eval/output_formatter_test.rb +++ b/test/agent_eval/output_formatter_test.rb @@ -125,6 +125,35 @@ def test_format_human_with_delta_report_fail assert_includes output, 'VERDICT: FAIL' end + def test_format_human_renders_tokens_and_cost + report = build_delta_report(verdict: true) + result = { + success: true, + eval_name: 'test-eval', + response: { report: report }, + tokens: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, + cost: 0.0125 + } + output = OutputFormatter.format(result, format: :human) + + assert_includes output, 'Tokens: 150' + assert_includes output, 'Est. Cost: $0.0125' + end + + def test_format_human_renders_dash_cost_when_unknown + report = build_delta_report(verdict: true) + result = { + success: true, + response: { report: report }, + tokens: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + cost: nil + } + output = OutputFormatter.format(result, format: :human) + + assert_includes output, 'Tokens: 0' + assert_includes output, 'Est. Cost: —' + end + def test_exit_code_returns_0_for_delta_report_pass result = { success: true, response: { report: build_delta_report(verdict: true) } } diff --git a/test/agent_eval/services/agent_spawner_service_test.rb b/test/agent_eval/services/agent_spawner_service_test.rb index cfd4b2b..d8d23c0 100644 --- a/test/agent_eval/services/agent_spawner_service_test.rb +++ b/test/agent_eval/services/agent_spawner_service_test.rb @@ -19,6 +19,52 @@ def test_call_returns_mock_result_for_mock_provider assert_equal 'mock result', result[:result] assert_equal [], result[:iterations] end + + def test_call_returns_zero_usage_for_mock_provider + result = AgentSpawnerService.call(@evaluation, @system_prompt, @provider, @config) + + assert_equal({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, result[:usage]) + end + + def test_call_surfaces_aggregated_agent_usage_for_real_provider + tmp = Dir.mktmpdir('agent_spawner_test') + begin + File.write(File.join(tmp, 'app.rb'), "# frozen_string_literal: true\n") + evaluation = Struct.new(:task, :path).new('Test task', tmp) + provider = Struct.new(:name, :runtime, :llm, :merged_config).new('openai', 'openai', 'gpt-4o', {}) + config = { api_key: 'fake-key', model: 'gpt-4o' } + + Agent::ReactAgent.stubs(:call).returns( + { + success: true, + response: { + content: 'done', + iterations: [], + usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 } + } + } + ) + Execution::Sandbox.stubs(:capture_diff).returns('No code changes made.') + + result = AgentSpawnerService.call(evaluation, @system_prompt, provider, config) + + assert_equal :success, result[:status] + assert_equal({ prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, result[:usage]) + ensure + FileUtils.rm_rf(tmp) + end + end + + def test_call_returns_zero_usage_on_rescue + evaluation = Struct.new(:task, :path).new('Test task', '/nonexistent/path/for/spawn') + provider = Struct.new(:name, :runtime, :llm, :merged_config).new('openai', 'openai', 'gpt-4o', {}) + config = { api_key: 'fake-key', model: 'gpt-4o' } + + result = AgentSpawnerService.call(evaluation, @system_prompt, provider, config) + + assert_equal :error, result[:status] + assert_equal({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, result[:usage]) + end end end end diff --git a/test/agent_eval/services/runner_service_test.rb b/test/agent_eval/services/runner_service_test.rb index ca98f54..f94b7c5 100644 --- a/test/agent_eval/services/runner_service_test.rb +++ b/test/agent_eval/services/runner_service_test.rb @@ -84,6 +84,33 @@ def test_success_result_includes_metadata assert_equal 'mock', result[:provider_name] end + def test_success_result_includes_tokens_and_cost_for_mock + write_mock_config + + SkillBench::Evaluation::Runner.expects(:call).returns({ + success: true, + response: { + report: Struct.new(:verdict, :baseline_total, :context_total, :deltas, + keyword_init: true).new( + verdict: true, + baseline_total: 30, + context_total: 80, + deltas: { 'correctness' => 16 } + ) + } + }) + + result = RunnerService.call( + eval_name: 'test-eval', + skill_names: ['test-skill'] + ) + + assert result[:success] + # Mock runs return no real usage: tokens sum to zero and cost is unknown. + assert_equal({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, result[:tokens]) + assert_nil result[:cost] + end + def test_call_raises_when_eval_not_found write_mock_config diff --git a/test/evaluator/react_agent/loop_runner_test.rb b/test/evaluator/react_agent/loop_runner_test.rb index 6511bce..a05ce94 100644 --- a/test/evaluator/react_agent/loop_runner_test.rb +++ b/test/evaluator/react_agent/loop_runner_test.rb @@ -151,5 +151,45 @@ def test_call_returns_iterations_on_max_iterations_error assert result[:response][:iterations] assert_equal 2, result[:response][:iterations].length end + + def test_call_accumulates_usage_across_iterations + Client.expects(:call).twice.returns( + { + success: true, + usage: { prompt_tokens: 10, completion_tokens: 4, total_tokens: 14 }, + response: { + message: { + 'content' => 'Thinking...', + 'tool_calls' => [{ 'id' => 'call_1', 'function' => { 'name' => 'run_command', 'arguments' => '{"command":"echo 1"}' } }] + } + } + } + ).then.returns( + { + success: true, + usage: { prompt_tokens: 6, completion_tokens: 2, total_tokens: 8 }, + response: { message: { 'content' => 'Final Answer', 'tool_calls' => [] } } + } + ) + + Tools.stubs(:execute).returns('Tool Result') + + result = Agent::ReactAgent::LoopRunner.call('Initial', 10, { working_dir: Dir.pwd, client_params: {} }) + usage = result[:response][:usage] + + assert_equal 16, usage[:prompt_tokens] + assert_equal 6, usage[:completion_tokens] + assert_equal 22, usage[:total_tokens] + end + + def test_call_usage_defaults_to_zero_without_client_usage + Client.expects(:call).returns( + { success: true, response: { message: { 'content' => 'Final Answer' } } } + ) + + result = Agent::ReactAgent::LoopRunner.call('Initial', 10, { client_params: {} }) + + assert_equal({ prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, result[:response][:usage]) + end end end diff --git a/test/evaluator/react_agent/step_test.rb b/test/evaluator/react_agent/step_test.rb index e211a3e..b12829f 100644 --- a/test/evaluator/react_agent/step_test.rb +++ b/test/evaluator/react_agent/step_test.rb @@ -127,6 +127,52 @@ def test_call_returns_iteration_metadata_on_client_failure assert_equal [], result[:iteration][:tools_used] assert_equal 'API Error', result[:iteration][:observation_summary] end + + def test_call_includes_usage_on_finish + Client.expects(:call).returns( + { + success: true, + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + response: { message: { 'content' => 'Final Answer', 'tool_calls' => [] } } + } + ) + + result = Agent::ReactAgent::Step.call(@messages, @config) + + assert_equal({ prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, result[:usage]) + end + + def test_call_includes_usage_when_executing_tools + Client.expects(:call).returns( + { + success: true, + usage: { prompt_tokens: 7, completion_tokens: 3, total_tokens: 10 }, + response: { + message: { + 'content' => 'Thinking...', + 'tool_calls' => [{ 'id' => 'call_1', 'function' => { 'name' => 'read_file', 'arguments' => '{"path":"a"}' } }] + } + } + } + ) + + Agent::ReactAgent::ToolExecutor.expects(:call).returns([{ role: 'tool', tool_call_id: 'call_1', content: 'result' }]) + + result = Agent::ReactAgent::Step.call(@messages, @config) + + assert result[:continue] + assert_equal({ prompt_tokens: 7, completion_tokens: 3, total_tokens: 10 }, result[:usage]) + end + + def test_call_usage_defaults_to_empty_hash_when_client_omits_it + Client.expects(:call).returns( + { success: true, response: { message: { 'content' => 'Final Answer', 'tool_calls' => [] } } } + ) + + result = Agent::ReactAgent::Step.call(@messages, @config) + + assert_equal({}, result[:usage]) + end end end end diff --git a/test/services/cost_calculator_test.rb b/test/services/cost_calculator_test.rb new file mode 100644 index 0000000..0f5c89b --- /dev/null +++ b/test/services/cost_calculator_test.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'test_helper' + +module SkillBench + module Services + class CostCalculatorTest < Minitest::Test + def test_computes_cost_for_known_model + usage = { prompt_tokens: 1000, completion_tokens: 2000, total_tokens: 3000 } + + cost = CostCalculator.call(usage: usage, model: 'gpt-4o') + + # 1K input * $0.005 + 2K output * $0.015 = 0.005 + 0.030 + assert_in_delta 0.035, cost, 1e-6 + end + + def test_computes_cost_for_anthropic_model + usage = { prompt_tokens: 1000, completion_tokens: 1000, total_tokens: 2000 } + + cost = CostCalculator.call(usage: usage, model: 'claude-3-5-sonnet') + + # 1K input * $0.003 + 1K output * $0.015 = 0.018 + assert_in_delta 0.018, cost, 1e-6 + end + + def test_matches_dated_model_variant_by_prefix + usage = { prompt_tokens: 1000, completion_tokens: 0, total_tokens: 1000 } + + cost = CostCalculator.call(usage: usage, model: 'claude-3-5-sonnet-20241022') + + assert_in_delta 0.003, cost, 1e-6 + end + + def test_prefers_longest_matching_prefix + usage = { prompt_tokens: 1000, completion_tokens: 0, total_tokens: 1000 } + + cost = CostCalculator.call(usage: usage, model: 'gpt-4o-mini-2024-07-18') + + # Must resolve to gpt-4o-mini ($0.00015), not gpt-4o ($0.005). + assert_in_delta 0.00015, cost, 1e-8 + end + + def test_returns_nil_for_unknown_model + usage = { prompt_tokens: 1000, completion_tokens: 1000, total_tokens: 2000 } + + assert_nil CostCalculator.call(usage: usage, model: 'mock') + end + + def test_returns_nil_for_nil_model + assert_nil CostCalculator.call(usage: { prompt_tokens: 10 }, model: nil) + end + + def test_handles_empty_usage_as_zero + cost = CostCalculator.call(usage: {}, model: 'gpt-4o') + + assert_in_delta 0.0, cost, 1e-9 + end + + def test_handles_nil_usage_without_crashing + cost = CostCalculator.call(usage: nil, model: 'gpt-4o') + + assert_in_delta 0.0, cost, 1e-9 + end + + def test_tolerates_string_usage_keys + usage = { 'prompt_tokens' => 1000, 'completion_tokens' => 0 } + + cost = CostCalculator.call(usage: usage, model: 'gpt-4o') + + assert_in_delta 0.005, cost, 1e-6 + end + end + end +end diff --git a/test/services/json_formatter_test.rb b/test/services/json_formatter_test.rb index 9a539ae..c66e9df 100644 --- a/test/services/json_formatter_test.rb +++ b/test/services/json_formatter_test.rb @@ -33,6 +33,25 @@ def test_format_is_pretty_printed assert_includes output, ' "a": 1' assert_includes output, '}' end + + def test_format_includes_tokens_and_cost + result = { + eval_name: 'e', + tokens: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, + cost: 0.0125 + } + parsed = JSON.parse(JsonFormatter.format(result)) + + assert_equal 150, parsed['tokens']['total_tokens'] + assert_in_delta(0.0125, parsed['cost']) + end + + def test_format_defaults_tokens_and_cost_when_absent + parsed = JSON.parse(JsonFormatter.format({ eval_name: 'e' })) + + assert_equal 0, parsed['tokens']['total_tokens'] + assert_nil parsed['cost'] + end end end end From 06015d7ef1082ea3742864f0a34de83105a5e27c Mon Sep 17 00:00:00 2001 From: Ismael G Marin C Date: Tue, 30 Jun 2026 14:53:52 -0600 Subject: [PATCH 2/2] fix(cost_calculator): price Claude 4 default models PRICES lacked the claude-sonnet-4 / claude-opus-4 families that the project actually passes in (config/defaults.rb -> claude-sonnet-4-20250514, clients/provider_schemas.rb -> claude-opus-4-7), so price_for fell through to nil and cost was reported unknown for the default Anthropic model. Add both families, preserving longest-prefix matching so dated variants still resolve. Covered by a new test asserting the real default strings. --- lib/skill_bench/services/cost_calculator.rb | 4 +++- test/services/cost_calculator_test.rb | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/skill_bench/services/cost_calculator.rb b/lib/skill_bench/services/cost_calculator.rb index addf8ac..64ab70c 100644 --- a/lib/skill_bench/services/cost_calculator.rb +++ b/lib/skill_bench/services/cost_calculator.rb @@ -10,7 +10,7 @@ module Services class CostCalculator # Approximate per-model prices in USD per 1,000 tokens. # Keyed by a canonical model prefix; longer prefixes win on lookup so that - # dated variants (e.g. "claude-3-5-sonnet-20241022") resolve correctly. + # dated variants (e.g. "claude-sonnet-4-20250514") resolve correctly. # Source: public OpenAI and Anthropic pricing pages (approximate). PRICES = { 'gpt-4o-mini' => { input: 0.00015, output: 0.0006 }, @@ -18,6 +18,8 @@ class CostCalculator 'gpt-4-turbo' => { input: 0.01, output: 0.03 }, 'gpt-4' => { input: 0.03, output: 0.06 }, 'gpt-3.5-turbo' => { input: 0.0005, output: 0.0015 }, + 'claude-opus-4' => { input: 0.015, output: 0.075 }, + 'claude-sonnet-4' => { input: 0.003, output: 0.015 }, 'claude-3-5-sonnet' => { input: 0.003, output: 0.015 }, 'claude-3-5-haiku' => { input: 0.0008, output: 0.004 }, 'claude-3-opus' => { input: 0.015, output: 0.075 }, diff --git a/test/services/cost_calculator_test.rb b/test/services/cost_calculator_test.rb index 0f5c89b..c8c0a15 100644 --- a/test/services/cost_calculator_test.rb +++ b/test/services/cost_calculator_test.rb @@ -31,6 +31,20 @@ def test_matches_dated_model_variant_by_prefix assert_in_delta 0.003, cost, 1e-6 end + def test_prices_claude_4_default_models + usage = { prompt_tokens: 1000, completion_tokens: 1000, total_tokens: 2000 } + + # Defaults the codebase actually passes in (config/defaults.rb, + # clients/provider_schemas.rb) must resolve, not fall through to nil. + sonnet_cost = CostCalculator.call(usage: usage, model: 'claude-sonnet-4-20250514') + opus_cost = CostCalculator.call(usage: usage, model: 'claude-opus-4-7') + + # claude-sonnet-4: 1K * $0.003 + 1K * $0.015 = 0.018 + assert_in_delta 0.018, sonnet_cost, 1e-6 + # claude-opus-4: 1K * $0.015 + 1K * $0.075 = 0.090 + assert_in_delta 0.090, opus_cost, 1e-6 + end + def test_prefers_longest_matching_prefix usage = { prompt_tokens: 1000, completion_tokens: 0, total_tokens: 1000 }