feat: per-run token and cost accounting (#40)

lib/skill_bench/agent/react_agent/loop_runner.rb

@@ -16,6 +16,7 @@ class LoopRunner
         def self.call(initial_prompt, max_iterations, config)
           messages = [{ role: 'user', content: initial_prompt }]
           iterations_log = []
+          total_usage = empty_usage
           step_count = 0
 
           while step_count < max_iterations
@@ -24,24 +25,27 @@ def self.call(initial_prompt, max_iterations, config)
             step_result = Step.call(messages, config)
             iteration = step_result[:iteration]
             iterations_log << attach_step_number(iteration, step_count) if iteration
+            total_usage = add_usage(total_usage, step_result[:usage])
 
             unless step_result[:continue]
               final_result = step_result[:result] || { success: false, response: { error: { message: 'Step returned no result' } } }
-              return merge_iterations(final_result, iterations_log)
+              return finalize(final_result, iterations_log, total_usage)
             end
 
             messages = step_result[:messages]
           end
 
-          merge_iterations(
+          finalize(
             { success: false, response: { error: { message: Agent::ReactAgent::MAX_ITERATIONS_REACHED } } },
-            iterations_log
+            iterations_log,
+            total_usage
           )
         rescue StandardError => e
           SkillBench::ErrorLogger.log_error(e, 'ReactAgent Error')
-          merge_iterations(
+          finalize(
             { success: false, response: { error: { message: e.message } } },
-            iterations_log
+            iterations_log,
+            total_usage
           )
         end
 
@@ -54,14 +58,45 @@ def self.attach_step_number(iteration, step_count)
           iteration.merge(step_number: step_count)
         end
 
-        # Merges the collected iterations into the result response.
+        # Merges the collected iterations and accumulated usage into the response.
         #
         # @param result [Hash] The final result hash from the loop.
         # @param iterations_log [Array<Hash>] Collected iteration metadata.
-        # @return [Hash] The result with :iterations injected into :response.
-        def self.merge_iterations(result, iterations_log)
+        # @param total_usage [Hash] Summed token usage across all iterations.
+        # @return [Hash] The result with :iterations and :usage injected into :response.
+        def self.finalize(result, iterations_log, total_usage)
           response = result[:response] || {}
-          result.merge(response: response.merge(iterations: iterations_log))
+          result.merge(response: response.merge(iterations: iterations_log, usage: total_usage))
+        end
+
+        # A zeroed token-usage accumulator.
+        #
+        # @return [Hash] Usage hash with prompt/completion/total token counts set to zero.
+        def self.empty_usage
+          { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }
+        end
+
+        # Adds a single step's usage onto a running total.
+        #
+        # @param total [Hash] The running usage total.
+        # @param usage [Hash, nil] A step's usage hash (may be nil or empty).
+        # @return [Hash] A new summed usage hash.
+        def self.add_usage(total, usage)
+          usage ||= {}
+          {
+            prompt_tokens: total[:prompt_tokens] + token_count(usage, :prompt_tokens),
+            completion_tokens: total[:completion_tokens] + token_count(usage, :completion_tokens),
+            total_tokens: total[:total_tokens] + token_count(usage, :total_tokens)
+          }
+        end
+
+        # Reads a token count from a usage hash, tolerating string keys.
+        #
+        # @param usage [Hash] The usage hash.
+        # @param key [Symbol] The usage key (e.g. :prompt_tokens).
+        # @return [Integer] The token count, or zero when absent.
+        def self.token_count(usage, key)
+          (usage[key] || usage[key.to_s] || 0).to_i
         end
       end
     end

lib/skill_bench/agent/react_agent/step.rb

@@ -12,7 +12,8 @@ class Step
         #
         # @param messages [Array<Hash>] The conversation history.
         # @param config [Hash] Configuration for this step (client params, system prompt, working dir).
-        # @return [Hash] Step outcome containing :continue (boolean), :result (hash, if finished), and :messages.
+        # @return [Hash] Step outcome containing :continue (boolean), :result (hash, if finished),
+        #   :usage (token usage for this step), and :messages.
         def self.call(messages, config)
           messages = messages.dup
           client_result = Client.call(
@@ -21,12 +22,14 @@ def self.call(messages, config)
             tools: Tools.definitions,
             **config[:client_params]
           )
+          usage = client_result[:usage] || {}
 
           unless client_result[:success]
             error_msg = client_result.dig(:response, :error, :message) || 'Unknown error'
             return {
               continue: false,
               result: client_result,
+              usage: usage,
               iteration: build_iteration(thought: '', tools_used: [], observation_summary: error_msg)
             }
           end
@@ -36,6 +39,7 @@ def self.call(messages, config)
             return {
               continue: false,
               result: { success: false, response: { error: { message: 'Empty response from LLM' } } },
+              usage: usage,
               iteration: build_iteration(thought: '', tools_used: [], observation_summary: 'Empty response from LLM')
             }
           end
@@ -51,6 +55,7 @@ def self.call(messages, config)
             return {
               continue: false,
               result: { success: true, response: { content: content } },
+              usage: usage,
               iteration: build_iteration(thought: thought, tools_used: [], observation_summary: '')
             }
           end
@@ -69,6 +74,7 @@ def self.call(messages, config)
           {
             continue: true,
             messages: messages,
+            usage: usage,
             iteration: build_iteration(thought: thought, tools_used: tools_used, observation_summary: observation_summary)
           }
         end

lib/skill_bench/output_formatter.rb

@@ -135,6 +135,7 @@ def self.format_delta_report(result, report)
         "  Eval: #{result[:eval_name] || ''}",
         "  Skill: #{result[:skill_name] || ''}",
         "  Provider: #{result[:provider_name] || ''}",
+        build_usage_line(result),
         ('═' * 55),
         ''
       ]
@@ -152,6 +153,19 @@ def self.format_delta_report(result, report)
     end
     private_class_method :format_delta_report
 
+    # Builds the token/cost summary line for the report header.
+    #
+    # @param result [Hash] Eval result envelope; reads :tokens and :cost.
+    # @return [String] A formatted "Tokens / Est. Cost" line.
+    def self.build_usage_line(result)
+      tokens = result[:tokens] || {}
+      total = tokens[:total_tokens] || tokens['total_tokens'] || 0
+      cost = result[:cost]
+      cost_label = cost ? Kernel.format('$%.4f', cost) : '—'
+      "  Tokens: #{total}  |  Est. Cost: #{cost_label}"
+    end
+    private_class_method :build_usage_line
+
     # Builds iteration timeline lines from the result response.
     #
     # @param result [Hash] Eval result envelope.

lib/skill_bench/services/agent_spawner_service.rb

@@ -7,6 +7,9 @@ module SkillBench
   module Services
     # Spawns and executes LLM agents for evaluation.
     class AgentSpawnerService
+      # Zeroed token usage used when a run produces no usage data (e.g. mock, rescue).
+      EMPTY_USAGE = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }.freeze
+
       # Spawns the LLM agent with the given system prompt.
       #
       # @param evaluation [SkillBench::Models::Eval] The eval being run
@@ -33,7 +36,7 @@ def initialize(evaluation, system_prompt, provider, config)
       #
       # @return [Hash] Agent response with result, status, runtime, usage, raw_response, iterations
       def call
-        return { result: 'mock result', status: :success, iterations: [] } if @provider.name == 'mock'
+        return { result: 'mock result', status: :success, iterations: [], usage: EMPTY_USAGE } if @provider.name == 'mock'
 
         client_params = build_client_params
         max_iterations = @config&.[](:max_iterations) || @config&.[]('max_iterations') || 25
@@ -63,14 +66,15 @@ def run_agent(client_params, max_iterations)
           final_answer = agent_result.dig(:response, :content) || ''
           diff = Execution::Sandbox.capture_diff(sandbox.path)
           iterations = agent_result.dig(:response, :iterations) || []
+          usage = agent_result.dig(:response, :usage) || EMPTY_USAGE
 
           output = [final_answer, diff].reject(&:empty?).join("\n\n")
 
           {
             result: output,
             status: status,
             runtime: @provider.runtime,
-            usage: {},
+            usage: usage,
             raw_response: agent_result,
             iterations: iterations
           }
@@ -80,7 +84,7 @@ def run_agent(client_params, max_iterations)
           result: "Error: #{e.message}",
           status: :error,
           runtime: @provider.runtime,
-          usage: {},
+          usage: EMPTY_USAGE,
           raw_response: { error: e.message, backtrace: e.backtrace },
           iterations: []
         }

lib/skill_bench/services/cost_calculator.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Estimates the USD cost of an LLM run from token usage and a model name.
+    #
+    # Prices are approximate, drawn from public OpenAI/Anthropic pricing pages,
+    # and expressed in USD per 1,000 tokens. Provider pricing changes over time,
+    # so treat the result as a rough estimate and extend {PRICES} as needed.
+    class CostCalculator
+      # Approximate per-model prices in USD per 1,000 tokens.
+      # Keyed by a canonical model prefix; longer prefixes win on lookup so that
+      # dated variants (e.g. "claude-sonnet-4-20250514") resolve correctly.
+      # Source: public OpenAI and Anthropic pricing pages (approximate).
+      PRICES = {
+        'gpt-4o-mini' => { input: 0.00015, output: 0.0006 },
+        'gpt-4o' => { input: 0.005, output: 0.015 },
+        'gpt-4-turbo' => { input: 0.01, output: 0.03 },
+        'gpt-4' => { input: 0.03, output: 0.06 },
+        'gpt-3.5-turbo' => { input: 0.0005, output: 0.0015 },
+        'claude-opus-4' => { input: 0.015, output: 0.075 },
+        'claude-sonnet-4' => { input: 0.003, output: 0.015 },
+        'claude-3-5-sonnet' => { input: 0.003, output: 0.015 },
+        'claude-3-5-haiku' => { input: 0.0008, output: 0.004 },
+        'claude-3-opus' => { input: 0.015, output: 0.075 },
+        'claude-3-sonnet' => { input: 0.003, output: 0.015 },
+        'claude-3-haiku' => { input: 0.00025, output: 0.00125 }
+      }.freeze
+
+      # Token count that one priced unit of {PRICES} covers.
+      TOKENS_PER_UNIT = 1000.0
+
+      # Estimates the USD cost for a run.
+      #
+      # @param usage [Hash, nil] Token usage with :prompt_tokens and :completion_tokens.
+      # @param model [String, nil] The model name (e.g. "gpt-4o").
+      # @return [Float, nil] Estimated cost in USD, or nil when the model is unknown.
+      def self.call(usage:, model:)
+        new(usage, model).call
+      end
+
+      # @param usage [Hash, nil] Token usage hash.
+      # @param model [String, nil] The model name.
+      def initialize(usage, model)
+        @usage = usage || {}
+        @model = model
+      end
+
+      # Estimates the USD cost for the configured usage and model.
+      #
+      # @return [Float, nil] Estimated cost in USD, or nil when the model is unknown.
+      def call
+        price = price_for(@model)
+        return nil unless price
+
+        input_cost = units(:prompt_tokens) * price[:input]
+        output_cost = units(:completion_tokens) * price[:output]
+        (input_cost + output_cost).round(6)
+      end
+
+      private
+
+      # Finds the price entry for a model by longest matching name prefix.
+      #
+      # @param model [String, nil] The model name.
+      # @return [Hash, nil] Price entry with :input and :output, or nil when unknown.
+      def price_for(model)
+        key = model.to_s.downcase
+        return PRICES[key] if PRICES.key?(key)
+
+        PRICES.select { |name, _| key.start_with?(name) }.max_by { |name, _| name.length }&.last
+      end
+
+      # Converts a usage token count into priced 1K-token units.
+      #
+      # @param key [Symbol] The usage key to read.
+      # @return [Float] The number of priced units.
+      def units(key)
+        token_count(key) / TOKENS_PER_UNIT
+      end
+
+      # Reads a token count from the usage hash, tolerating string keys.
+      #
+      # @param key [Symbol] The usage key (e.g. :prompt_tokens).
+      # @return [Integer] The token count, or zero when absent.
+      def token_count(key)
+        (@usage[key] || @usage[key.to_s] || 0).to_i
+      end
+    end
+  end
+end

lib/skill_bench/services/json_formatter.rb

@@ -6,12 +6,32 @@ module SkillBench
   module Services
     # Formats evaluation results as JSON.
     class JsonFormatter
+      # Zeroed token usage used when a result carries no usage data.
+      EMPTY_USAGE = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }.freeze
+
       # Format result as JSON.
       #
+      # Ensures top-level :tokens and :cost fields are always present (additive;
+      # existing keys are preserved) so JSON consumers see a stable shape.
+      #
       # @param result [Hash] Eval result.
       # @return [String] JSON-formatted string.
       def self.format(result)
-        JSON.pretty_generate(result)
+        JSON.pretty_generate(with_usage_fields(result))
+      end
+
+      # Returns the result augmented with token/cost fields when missing.
+      #
+      # @param result [Hash] Eval result (returned unchanged when not a Hash).
+      # @return [Hash] Result with :tokens and :cost guaranteed present.
+      def self.with_usage_fields(result)
+        return result unless result.is_a?(Hash)
+
+        defaults = {
+          tokens: result[:tokens] || result.dig(:response, :tokens) || EMPTY_USAGE,
+          cost: result.key?(:cost) ? result[:cost] : result.dig(:response, :cost)
+        }
+        defaults.merge(result)
       end
     end
   end