From 1c11b7d8427de9880a11f2e0953722e27da32082 Mon Sep 17 00:00:00 2001 From: eileencodes Date: Thu, 30 Apr 2026 13:08:03 -0400 Subject: [PATCH] Allow declaring override remotes on a RubyGems source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `source` may declare one or more `overrides:` — secondary RubyGems-compatible repositories that supply alternate builds (for example, prebuilt binaries) of gems that already exist in the primary source. source "https://rubygems.org", overrides: ["https://build-farm.example.com"] Override remotes are consulted only for gems that are also present in the primary source; they cannot introduce new gems. When an override publishes a spec whose name and version match one in the primary source, that spec is preferred for installation if it is compatible with the local platform. If no matching build is available, or fetching the override fails (authentication, SSL, or network errors), Bundler falls back to the primary source. Override remotes are recorded in `Gemfile.lock` under `override:` lines: GEM remote: https://rubygems.org/ override: https://build-farm.example.com/ specs: IRL Testing: This can be exercised against Kou's precompiled-gems build farm hosted on Cloudsmith. The only build target currently published there is Ruby 4.0 on amd64 Ubuntu 24.04 — on any other platform the override is skipped and Bundler falls back to the primary source. source "https://rubygems.org", overrides: ["https://dl.cloudsmith.io/public/rubygems-precompiled-gems/ruby-4-0-amd64-ubuntu-24-04/ruby/"] # Precompiled binaries are available on the override remote: gem "json" gem "openc3" gem "io-event" # Not in the override; fall back to rubygems.org and compile # from source: gem "nokogiri" gem "rails" --- bundler/lib/bundler/dsl.rb | 8 +- bundler/lib/bundler/man/gemfile.5.ronn | 27 ++++ bundler/lib/bundler/source/rubygems.rb | 96 ++++++++++- bundler/lib/bundler/source_list.rb | 5 +- spec/bundler/lockfile_parser_spec.rb | 63 ++++++++ spec/bundler/source/rubygems_spec.rb | 213 +++++++++++++++++++++++++ spec/bundler/source_list_spec.rb | 32 ++++ 7 files changed, 437 insertions(+), 7 deletions(-) diff --git a/bundler/lib/bundler/dsl.rb b/bundler/lib/bundler/dsl.rb index 6f06c4e91879..9af78a0f638d 100644 --- a/bundler/lib/bundler/dsl.rb +++ b/bundler/lib/bundler/dsl.rb @@ -116,6 +116,7 @@ def source(source, *args, &blk) options = args.last.is_a?(Hash) ? args.pop.dup : {} options = normalize_hash(options) source = normalize_source(source) + overrides = options["overrides"] if options.key?("type") options["type"] = options["type"].to_s @@ -130,9 +131,12 @@ def source(source, *args, &blk) source_opts = options.merge("uri" => source) with_source(@sources.add_plugin_source(options["type"], source_opts), &blk) elsif block_given? - with_source(@sources.add_rubygems_source("remotes" => source), &blk) + with_source(@sources.add_rubygems_source( + "remotes" => source, + "overrides" => overrides + ), &blk) else - @sources.add_global_rubygems_remote(source) + @sources.add_global_rubygems_remote(source, overrides: overrides) end end diff --git a/bundler/lib/bundler/man/gemfile.5.ronn b/bundler/lib/bundler/man/gemfile.5.ronn index 18d7bb826e44..5347d0bdfddf 100644 --- a/bundler/lib/bundler/man/gemfile.5.ronn +++ b/bundler/lib/bundler/man/gemfile.5.ronn @@ -55,6 +55,33 @@ include the credentials in the Gemfile as part of the source URL. Credentials in the source URL will take precedence over credentials set using `config`. +### OVERRIDES + +A `source` may declare one or more `overrides`. An override is a secondary +RubyGems-compatible repository that supplies alternate builds (for example, +prebuilt binaries) of gems that already exist in the primary source. + + source "https://rubygems.org", + overrides: ["https://build-farm.example.com"] + +Overrides are consulted only for gems that are also present in the primary +source — they cannot introduce new gems. If an override publishes a gem that +the primary source does not contain in any version, Bundler raises an error, +since this indicates the override and primary source are misconfigured. + +When an override publishes a spec whose name and version match one in the +primary source, that spec is preferred for installation if it is compatible +with the local platform. If no matching build is available, or fetching the +override fails (authentication, SSL, network), Bundler falls back to the +primary source. + +Overrides are recorded in `Gemfile.lock` under `override:` lines: + + GEM + remote: https://rubygems.org/ + override: https://build-farm.example.com/ + specs: + ## RUBY If your application requires a specific Ruby version or engine, specify your diff --git a/bundler/lib/bundler/source/rubygems.rb b/bundler/lib/bundler/source/rubygems.rb index b5c3b9169d16..3c019a70bf51 100644 --- a/bundler/lib/bundler/source/rubygems.rb +++ b/bundler/lib/bundler/source/rubygems.rb @@ -12,10 +12,12 @@ class Rubygems < Source REQUIRE_MUTEX = Mutex.new attr_accessor :remotes + attr_reader :override_remotes def initialize(options = {}) @options = options @remotes = [] + @override_remotes = [] @dependency_names = [] @allow_remote = false @allow_cached = false @@ -26,8 +28,12 @@ def initialize(options = {}) @gem_installers_mutex = Mutex.new Array(options["remotes"]).reverse_each {|r| add_remote(r) } + Array(options["overrides"]).reverse_each {|override| add_override_remote(override) } - @lockfile_remotes = @remotes if options["from_lockfile"] + if options["from_lockfile"] + @lockfile_remotes = @remotes + @lockfile_override_remotes = @override_remotes + end end def caches @@ -73,11 +79,13 @@ def cached! end def hash - @remotes.hash + [@remotes, @override_remotes].hash end def eql?(other) - other.is_a?(Rubygems) && other.credless_remotes == credless_remotes + other.is_a?(Rubygems) && + other.credless_remotes == credless_remotes && + other.override_remotes == override_remotes end alias_method :==, :eql? @@ -105,6 +113,7 @@ def options def self.from_lock(options) options["remotes"] = Array(options.delete("remote")).reverse + options["overrides"] = Array(options.delete("override")).reverse new(options.merge("from_lockfile" => true)) end @@ -113,6 +122,9 @@ def to_lock lockfile_remotes.reverse_each do |remote| out << " remote: #{remote}\n" end + lockfile_override_remotes.reverse_each do |override| + out << " override: #{override}\n" + end out << " specs:\n" end @@ -248,6 +260,11 @@ def add_remote(source) @remotes.unshift(uri) unless @remotes.include?(uri) end + def add_override_remote(source) + uri = normalize_uri(source) + @override_remotes.unshift(uri) unless @override_remotes.include?(uri) + end + def spec_names if dependency_api_available? remote_specs.spec_names @@ -275,6 +292,17 @@ def fetchers @fetchers ||= remote_fetchers.values.freeze end + def override_remote_fetchers + @override_remote_fetchers ||= @override_remotes.to_h do |uri| + remote = Source::Rubygems::Remote.new(uri) + [remote, Bundler::Fetcher.new(remote)] + end.freeze + end + + def override_fetchers + @override_fetchers ||= override_remote_fetchers.values.freeze + end + def double_check_for(unmet_dependency_names) return unless dependency_api_available? @@ -324,6 +352,10 @@ def credless_remotes remotes.map(&method(:remove_auth)) end + def credless_override_remotes + override_remotes.map(&method(:remove_auth)) + end + def cached_gem(spec) global_cache_path = download_cache_path(spec) caches << global_cache_path if global_cache_path @@ -399,6 +431,8 @@ def remote_specs else fetch_names(fetchers, nil, idx) end + + fetch_override_specs(idx) if @override_remotes.any? && @allow_remote end end @@ -415,6 +449,51 @@ def fetch_names(fetchers, dependency_names, index) end end + # Merge spec from override remotes into +idx+, but only for gems already + # present in the parent source. + def fetch_override_specs(idx) + local_platform = Bundler.local_platform + + override_fetchers.each do |fetcher| + filtered_uri = URICredentialsFilter.credential_filtered_uri(fetcher.uri) + begin + Bundler.ui.info "Fetching override gem metadata from #{filtered_uri}", Bundler.ui.debug? + override_index = fetcher.specs_with_retry(dependency_names, self) + Bundler.ui.info "" unless Bundler.ui.debug? + + override_index.each do |spec| + primary_specs = idx.search(spec.name) + if primary_specs.empty? + raise GemfileError, "Override source #{filtered_uri} provides #{spec.name}, but no version of #{spec.name} exists in the primary source. Override sources may only supply alternate builds for gems already present in the primary source." + end + + unless primary_specs.any? {|s| s.version == spec.version } + Bundler.ui.debug "Skipping #{spec.full_name} from override source #{filtered_uri} (no matching version in parent source)" + next + end + + unless spec.installable_on_platform?(local_platform) + Bundler.ui.debug "Skipping #{spec.full_name} from override source #{filtered_uri} (platform #{spec.platform} not compatible with #{local_platform})" + next + end + + idx << spec + end + rescue Bundler::Fetcher::AuthenticationRequiredError, Bundler::Fetcher::BadAuthenticationError, Bundler::Fetcher::AuthenticationForbiddenError => e + warn_override_failure(filtered_uri, "requires authentication", e) + rescue Bundler::Fetcher::CertificateFailureError, Bundler::Fetcher::SSLError => e + warn_override_failure(filtered_uri, "has SSL errors", e) + rescue Bundler::Fetcher::FallbackError, Bundler::HTTPError => e + warn_override_failure(filtered_uri, "is unreachable", e) + end + end + end + + def warn_override_failure(filtered_uri, reason, error) + Bundler.ui.warn "Override source #{filtered_uri} #{reason}: #{error.message}. Falling back to source compilation." + Bundler.ui.debug "#{error.class}: #{error.message}" + end + def fetch_gem_if_possible(spec, previous_spec = nil) if spec.remote fetch_gem(spec, previous_spec) @@ -460,6 +539,15 @@ def lockfile_remotes @lockfile_remotes || credless_remotes end + def lockfile_override_remotes + @lockfile_override_remotes || credless_override_remotes + end + + # Combined lookup for download_gem - includes both primary and override fetchers + def all_remote_fetchers + @all_remote_fetchers ||= remote_fetchers.merge(override_remote_fetchers).freeze + end + # Checks if the requested spec exists in the global cache. If it does, # we copy it to the download path, and if it does not, we download it. # @@ -475,7 +563,7 @@ def lockfile_remotes def download_gem(spec, download_cache_path, previous_spec = nil) uri = spec.remote.uri Bundler.ui.confirm("Fetching #{version_message(spec, previous_spec)}") - gem_remote_fetcher = remote_fetchers.fetch(spec.remote).gem_remote_fetcher + gem_remote_fetcher = all_remote_fetchers.fetch(spec.remote).gem_remote_fetcher Gem.time("Downloaded #{spec.name} in", 0, true) do Bundler.rubygems.download_gem(spec, uri, download_cache_path, gem_remote_fetcher) diff --git a/bundler/lib/bundler/source_list.rb b/bundler/lib/bundler/source_list.rb index 38fa0972e64e..0a77f60d23d3 100644 --- a/bundler/lib/bundler/source_list.rb +++ b/bundler/lib/bundler/source_list.rb @@ -59,8 +59,11 @@ def add_plugin_source(source, options = {}) add_source_to_list Plugin.source(source).new(options), @plugin_sources end - def add_global_rubygems_remote(uri) + def add_global_rubygems_remote(uri, overrides: nil) global_rubygems_source.add_remote(uri) + + overrides&.each {|override| global_rubygems_source.add_override_remote(override) } + global_rubygems_source end diff --git a/spec/bundler/lockfile_parser_spec.rb b/spec/bundler/lockfile_parser_spec.rb index f38da2c99321..be2233ec9b3b 100644 --- a/spec/bundler/lockfile_parser_spec.rb +++ b/spec/bundler/lockfile_parser_spec.rb @@ -164,6 +164,69 @@ include_examples "parsing" end + context "when a GEM source has override remotes" do + let(:lockfile_contents) { <<~L } + GEM + remote: https://rubygems.org/ + override: https://build-farm.example.com/ + specs: + rake (10.3.2) + + PLATFORMS + ruby + + DEPENDENCIES + rake + + CHECKSUMS + rake (10.3.2) sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8 + + BUNDLED WITH + 1.12.0.rc.2 + L + + before { allow(Bundler::SharedHelpers).to receive(:find_gemfile).and_return(bundled_app("gems.rb")) } + subject { described_class.new(lockfile_contents) } + + it "parses the override remote" do + gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) } + expect(gem_source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end + + it "preserves the primary remote" do + gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) } + expect(gem_source.remotes.map(&:to_s)).to include("https://rubygems.org/") + end + end + + context "when a GEM source has multiple override remotes" do + let(:lockfile_contents) { <<~L } + GEM + remote: https://rubygems.org/ + override: https://first.example.com/ + override: https://second.example.com/ + specs: + rake (10.3.2) + + PLATFORMS + ruby + + DEPENDENCIES + rake + + BUNDLED WITH + 1.12.0.rc.2 + L + + before { allow(Bundler::SharedHelpers).to receive(:find_gemfile).and_return(bundled_app("gems.rb")) } + subject { described_class.new(lockfile_contents) } + + it "parses multiple override remotes" do + gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) } + expect(gem_source.override_remotes.size).to eq 2 + end + end + context "when the checksum is urlsafe base64 encoded" do let(:lockfile_contents) do super().sub( diff --git a/spec/bundler/source/rubygems_spec.rb b/spec/bundler/source/rubygems_spec.rb index dde4e4ed4769..890928cceb9d 100644 --- a/spec/bundler/source/rubygems_spec.rb +++ b/spec/bundler/source/rubygems_spec.rb @@ -31,6 +31,34 @@ end end + describe "#add_override_remote" do + it "normalizes and stores the URI" do + subject.add_override_remote("https://build-farm.example.com") + expect(subject.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end + + it "prepends new override remotes" do + subject.add_override_remote("https://first.example.com") + subject.add_override_remote("https://second.example.com") + expect(subject.override_remotes).to eq [ + Gem::URI("https://second.example.com/"), + Gem::URI("https://first.example.com/"), + ] + end + + it "does not add duplicates" do + subject.add_override_remote("https://build-farm.example.com") + subject.add_override_remote("https://build-farm.example.com") + expect(subject.override_remotes.size).to eq 1 + end + + context "when the source is an HTTP(s) URI with no host" do + it "raises error" do + expect { subject.add_override_remote("https:build-farm.example.com") }.to raise_error(ArgumentError) + end + end + end + describe "#no_remotes?" do context "when no remote provided" do it "returns a truthy value" do @@ -45,6 +73,191 @@ end end + describe "override remotes in initialize" do + it "stores override remotes from options" do + source = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end + + it "handles multiple override remotes" do + source = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://first.example.com", "https://second.example.com"] + ) + expect(source.override_remotes.size).to eq 2 + end + + it "defaults to empty override remotes" do + source = described_class.new("remotes" => ["https://rubygems.org"]) + expect(source.override_remotes).to eq [] + end + end + + describe "#eql?" do + it "considers sources with different override remotes as not equal" do + without_overrides = described_class.new("remotes" => ["https://rubygems.org"]) + with_overrides = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(without_overrides).not_to eql(with_overrides) + end + + it "considers sources with same remotes and override remotes as equal" do + source_a = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + source_b = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(source_a).to eql(source_b) + end + end + + describe "#hash" do + it "differs for sources with different override remotes" do + without_overrides = described_class.new("remotes" => ["https://rubygems.org"]) + with_overrides = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(without_overrides.hash).not_to eq(with_overrides.hash) + end + end + + describe "#to_lock" do + it "includes override lines" do + source = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expected = <<~L + GEM + remote: https://rubygems.org/ + override: https://build-farm.example.com/ + specs: + L + expect(source.to_lock).to eq(expected) + end + + it "includes multiple override lines" do + source = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://first.example.com", "https://second.example.com"] + ) + lock = source.to_lock + expect(lock).to include("override: https://first.example.com/") + expect(lock).to include("override: https://second.example.com/") + end + + it "omits override lines when no override remotes" do + source = described_class.new("remotes" => ["https://rubygems.org"]) + expect(source.to_lock).not_to include("override:") + end + end + + describe ".from_lock" do + it "restores override remotes from lockfile options" do + source = described_class.from_lock( + "remote" => "https://rubygems.org/", + "override" => "https://build-farm.example.com/" + ) + expect(source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end + + it "restores multiple override remotes from lockfile options" do + source = described_class.from_lock( + "remote" => "https://rubygems.org/", + "override" => ["https://first.example.com/", "https://second.example.com/"] + ) + expect(source.override_remotes.size).to eq 2 + end + + it "handles missing override key" do + source = described_class.from_lock("remote" => "https://rubygems.org/") + expect(source.override_remotes).to eq [] + end + end + + describe "to_lock/from_lock round-trip" do + it "preserves override remotes" do + original = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + + # Simulate lockfile parser: parse the to_lock output + lock_output = original.to_lock + opts = {} + lock_output.each_line do |line| + next unless line =~ /^\s+([a-z]+): (.*)$/i + key = $1 + value = $2 + if opts[key] + opts[key] = Array(opts[key]) + opts[key] << value + else + opts[key] = value + end + end + + restored = described_class.from_lock(opts) + expect(restored.remotes.map(&:to_s)).to eq(original.remotes.map(&:to_s)) + expect(restored.override_remotes.map(&:to_s)).to eq(original.override_remotes.map(&:to_s)) + end + end + + describe "#fetch_override_specs" do + let(:source) do + described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + end + + let(:override_uri) { Gem::URI("https://build-farm.example.com/") } + + def stub_override_fetcher(specs) + override_index = Bundler::Index.new + Array(specs).each {|s| override_index << s } + fetcher = double(Bundler::Fetcher, uri: override_uri) + allow(fetcher).to receive(:specs_with_retry).and_return(override_index) + allow(source).to receive(:override_fetchers).and_return([fetcher]) + allow(source).to receive(:dependency_names).and_return([]) + end + + it "raises GemfileError when override provides a gem absent from primary" do + override_spec = Gem::Specification.new("missing-from-primary", "1.0.0") + stub_override_fetcher(override_spec) + + primary_idx = Bundler::Index.new + + expect do + source.send(:fetch_override_specs, primary_idx) + end.to raise_error(Bundler::GemfileError, /missing-from-primary/) + end + + it "skips override specs whose version does not match the primary source" do + primary_spec = Gem::Specification.new("json", "2.7.0") + override_spec = Gem::Specification.new("json", "2.7.1") + stub_override_fetcher(override_spec) + + primary_idx = Bundler::Index.new + primary_idx << primary_spec + + expect do + source.send(:fetch_override_specs, primary_idx) + end.not_to raise_error + + expect(primary_idx.search(["json", "2.7.1"])).to be_empty + end + end + describe "log debug information" do it "log the time spent downloading and installing a gem" do build_repo2 do diff --git a/spec/bundler/source_list_spec.rb b/spec/bundler/source_list_spec.rb index 6e0be6c92fcc..0aa69590f79d 100644 --- a/spec/bundler/source_list_spec.rb +++ b/spec/bundler/source_list_spec.rb @@ -129,6 +129,38 @@ Gem::URI("https://rubygems.org/"), ] end + + it "adds override remotes to the global source" do + source_list.add_global_rubygems_remote( + "https://rubygems.org", + overrides: ["https://build-farm.example.com"] + ) + expect(returned_source.override_remotes).to eq [ + Gem::URI("https://build-farm.example.com/"), + ] + end + + it "adds multiple override remotes to the global source" do + source_list.add_global_rubygems_remote( + "https://rubygems.org", + overrides: ["https://first.example.com", "https://second.example.com"] + ) + expect(returned_source.override_remotes.size).to eq 2 + end + + it "does not add override remotes when none are provided" do + expect(returned_source.override_remotes).to eq [] + end + end + + describe "#add_rubygems_source with overrides" do + it "passes override remotes to the new source" do + source = source_list.add_rubygems_source( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end end describe "#add_plugin_source" do