From dc52c783044dc63f85c745df1ff86422a50a0bd7 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Wed, 25 Mar 2026 10:19:22 -0700 Subject: [PATCH] Warn when two executables have the same revision and flags When benchmarking two rubies that produce identical `ruby -v` output and use the same command-line flags, it's likely a user mistake. Skip the warning when JIT flags differ, since comparing JIT vs non-JIT on the same revision is a valid use case. --- lib/benchmark_runner/cli.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index de857e67..634d207d 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -9,6 +9,9 @@ module BenchmarkRunner class CLI + BOLD = "\e[1m" + RESET = "\e[0m" + attr_reader :args def self.run(argv = ARGV) @@ -45,6 +48,18 @@ def run ruby_descriptions[name] = `#{executable.shelljoin} -v`.chomp end + # Warn if two executables look identical (same ruby -v output and same flags) + names = ruby_descriptions.keys + names.each_with_index do |name_a, i| + names[(i + 1)..].each do |name_b| + flags_a = args.executables[name_a][1..] || [] + flags_b = args.executables[name_b][1..] || [] + if ruby_descriptions[name_a] == ruby_descriptions[name_b] && flags_a == flags_b + warn "#{BOLD}WARNING: '#{name_a}' and '#{name_b}' appear identical (same revision, same flags). This is likely a mistake.#{RESET}" + end + end + end + bench_start_time = Time.now.to_f bench_data = {} bench_failures = {}