diff --git a/bundler/lib/bundler/dsl.rb b/bundler/lib/bundler/dsl.rb index 6f06c4e91879..9af78a0f638d 100644 --- a/bundler/lib/bundler/dsl.rb +++ b/bundler/lib/bundler/dsl.rb @@ -116,6 +116,7 @@ def source(source, *args, &blk) options = args.last.is_a?(Hash) ? args.pop.dup : {} options = normalize_hash(options) source = normalize_source(source) + overrides = options["overrides"] if options.key?("type") options["type"] = options["type"].to_s @@ -130,9 +131,12 @@ def source(source, *args, &blk) source_opts = options.merge("uri" => source) with_source(@sources.add_plugin_source(options["type"], source_opts), &blk) elsif block_given? - with_source(@sources.add_rubygems_source("remotes" => source), &blk) + with_source(@sources.add_rubygems_source( + "remotes" => source, + "overrides" => overrides + ), &blk) else - @sources.add_global_rubygems_remote(source) + @sources.add_global_rubygems_remote(source, overrides: overrides) end end diff --git a/bundler/lib/bundler/man/gemfile.5.ronn b/bundler/lib/bundler/man/gemfile.5.ronn index 18d7bb826e44..5347d0bdfddf 100644 --- a/bundler/lib/bundler/man/gemfile.5.ronn +++ b/bundler/lib/bundler/man/gemfile.5.ronn @@ -55,6 +55,33 @@ include the credentials in the Gemfile as part of the source URL. Credentials in the source URL will take precedence over credentials set using `config`. +### OVERRIDES + +A `source` may declare one or more `overrides`. An override is a secondary +RubyGems-compatible repository that supplies alternate builds (for example, +prebuilt binaries) of gems that already exist in the primary source. + + source "https://rubygems.org", + overrides: ["https://build-farm.example.com"] + +Overrides are consulted only for gems that are also present in the primary +source — they cannot introduce new gems. If an override publishes a gem that +the primary source does not contain in any version, Bundler raises an error, +since this indicates the override and primary source are misconfigured. + +When an override publishes a spec whose name and version match one in the +primary source, that spec is preferred for installation if it is compatible +with the local platform. If no matching build is available, or fetching the +override fails (authentication, SSL, network), Bundler falls back to the +primary source. + +Overrides are recorded in `Gemfile.lock` under `override:` lines: + + GEM + remote: https://rubygems.org/ + override: https://build-farm.example.com/ + specs: + ## RUBY If your application requires a specific Ruby version or engine, specify your diff --git a/bundler/lib/bundler/source/rubygems.rb b/bundler/lib/bundler/source/rubygems.rb index b5c3b9169d16..3c019a70bf51 100644 --- a/bundler/lib/bundler/source/rubygems.rb +++ b/bundler/lib/bundler/source/rubygems.rb @@ -12,10 +12,12 @@ class Rubygems < Source REQUIRE_MUTEX = Mutex.new attr_accessor :remotes + attr_reader :override_remotes def initialize(options = {}) @options = options @remotes = [] + @override_remotes = [] @dependency_names = [] @allow_remote = false @allow_cached = false @@ -26,8 +28,12 @@ def initialize(options = {}) @gem_installers_mutex = Mutex.new Array(options["remotes"]).reverse_each {|r| add_remote(r) } + Array(options["overrides"]).reverse_each {|override| add_override_remote(override) } - @lockfile_remotes = @remotes if options["from_lockfile"] + if options["from_lockfile"] + @lockfile_remotes = @remotes + @lockfile_override_remotes = @override_remotes + end end def caches @@ -73,11 +79,13 @@ def cached! end def hash - @remotes.hash + [@remotes, @override_remotes].hash end def eql?(other) - other.is_a?(Rubygems) && other.credless_remotes == credless_remotes + other.is_a?(Rubygems) && + other.credless_remotes == credless_remotes && + other.override_remotes == override_remotes end alias_method :==, :eql? @@ -105,6 +113,7 @@ def options def self.from_lock(options) options["remotes"] = Array(options.delete("remote")).reverse + options["overrides"] = Array(options.delete("override")).reverse new(options.merge("from_lockfile" => true)) end @@ -113,6 +122,9 @@ def to_lock lockfile_remotes.reverse_each do |remote| out << " remote: #{remote}\n" end + lockfile_override_remotes.reverse_each do |override| + out << " override: #{override}\n" + end out << " specs:\n" end @@ -248,6 +260,11 @@ def add_remote(source) @remotes.unshift(uri) unless @remotes.include?(uri) end + def add_override_remote(source) + uri = normalize_uri(source) + @override_remotes.unshift(uri) unless @override_remotes.include?(uri) + end + def spec_names if dependency_api_available? remote_specs.spec_names @@ -275,6 +292,17 @@ def fetchers @fetchers ||= remote_fetchers.values.freeze end + def override_remote_fetchers + @override_remote_fetchers ||= @override_remotes.to_h do |uri| + remote = Source::Rubygems::Remote.new(uri) + [remote, Bundler::Fetcher.new(remote)] + end.freeze + end + + def override_fetchers + @override_fetchers ||= override_remote_fetchers.values.freeze + end + def double_check_for(unmet_dependency_names) return unless dependency_api_available? @@ -324,6 +352,10 @@ def credless_remotes remotes.map(&method(:remove_auth)) end + def credless_override_remotes + override_remotes.map(&method(:remove_auth)) + end + def cached_gem(spec) global_cache_path = download_cache_path(spec) caches << global_cache_path if global_cache_path @@ -399,6 +431,8 @@ def remote_specs else fetch_names(fetchers, nil, idx) end + + fetch_override_specs(idx) if @override_remotes.any? && @allow_remote end end @@ -415,6 +449,51 @@ def fetch_names(fetchers, dependency_names, index) end end + # Merge spec from override remotes into +idx+, but only for gems already + # present in the parent source. + def fetch_override_specs(idx) + local_platform = Bundler.local_platform + + override_fetchers.each do |fetcher| + filtered_uri = URICredentialsFilter.credential_filtered_uri(fetcher.uri) + begin + Bundler.ui.info "Fetching override gem metadata from #{filtered_uri}", Bundler.ui.debug? + override_index = fetcher.specs_with_retry(dependency_names, self) + Bundler.ui.info "" unless Bundler.ui.debug? + + override_index.each do |spec| + primary_specs = idx.search(spec.name) + if primary_specs.empty? + raise GemfileError, "Override source #{filtered_uri} provides #{spec.name}, but no version of #{spec.name} exists in the primary source. Override sources may only supply alternate builds for gems already present in the primary source." + end + + unless primary_specs.any? {|s| s.version == spec.version } + Bundler.ui.debug "Skipping #{spec.full_name} from override source #{filtered_uri} (no matching version in parent source)" + next + end + + unless spec.installable_on_platform?(local_platform) + Bundler.ui.debug "Skipping #{spec.full_name} from override source #{filtered_uri} (platform #{spec.platform} not compatible with #{local_platform})" + next + end + + idx << spec + end + rescue Bundler::Fetcher::AuthenticationRequiredError, Bundler::Fetcher::BadAuthenticationError, Bundler::Fetcher::AuthenticationForbiddenError => e + warn_override_failure(filtered_uri, "requires authentication", e) + rescue Bundler::Fetcher::CertificateFailureError, Bundler::Fetcher::SSLError => e + warn_override_failure(filtered_uri, "has SSL errors", e) + rescue Bundler::Fetcher::FallbackError, Bundler::HTTPError => e + warn_override_failure(filtered_uri, "is unreachable", e) + end + end + end + + def warn_override_failure(filtered_uri, reason, error) + Bundler.ui.warn "Override source #{filtered_uri} #{reason}: #{error.message}. Falling back to source compilation." + Bundler.ui.debug "#{error.class}: #{error.message}" + end + def fetch_gem_if_possible(spec, previous_spec = nil) if spec.remote fetch_gem(spec, previous_spec) @@ -460,6 +539,15 @@ def lockfile_remotes @lockfile_remotes || credless_remotes end + def lockfile_override_remotes + @lockfile_override_remotes || credless_override_remotes + end + + # Combined lookup for download_gem - includes both primary and override fetchers + def all_remote_fetchers + @all_remote_fetchers ||= remote_fetchers.merge(override_remote_fetchers).freeze + end + # Checks if the requested spec exists in the global cache. If it does, # we copy it to the download path, and if it does not, we download it. # @@ -475,7 +563,7 @@ def lockfile_remotes def download_gem(spec, download_cache_path, previous_spec = nil) uri = spec.remote.uri Bundler.ui.confirm("Fetching #{version_message(spec, previous_spec)}") - gem_remote_fetcher = remote_fetchers.fetch(spec.remote).gem_remote_fetcher + gem_remote_fetcher = all_remote_fetchers.fetch(spec.remote).gem_remote_fetcher Gem.time("Downloaded #{spec.name} in", 0, true) do Bundler.rubygems.download_gem(spec, uri, download_cache_path, gem_remote_fetcher) diff --git a/bundler/lib/bundler/source_list.rb b/bundler/lib/bundler/source_list.rb index 38fa0972e64e..0a77f60d23d3 100644 --- a/bundler/lib/bundler/source_list.rb +++ b/bundler/lib/bundler/source_list.rb @@ -59,8 +59,11 @@ def add_plugin_source(source, options = {}) add_source_to_list Plugin.source(source).new(options), @plugin_sources end - def add_global_rubygems_remote(uri) + def add_global_rubygems_remote(uri, overrides: nil) global_rubygems_source.add_remote(uri) + + overrides&.each {|override| global_rubygems_source.add_override_remote(override) } + global_rubygems_source end diff --git a/spec/bundler/lockfile_parser_spec.rb b/spec/bundler/lockfile_parser_spec.rb index f38da2c99321..be2233ec9b3b 100644 --- a/spec/bundler/lockfile_parser_spec.rb +++ b/spec/bundler/lockfile_parser_spec.rb @@ -164,6 +164,69 @@ include_examples "parsing" end + context "when a GEM source has override remotes" do + let(:lockfile_contents) { <<~L } + GEM + remote: https://rubygems.org/ + override: https://build-farm.example.com/ + specs: + rake (10.3.2) + + PLATFORMS + ruby + + DEPENDENCIES + rake + + CHECKSUMS + rake (10.3.2) sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8 + + BUNDLED WITH + 1.12.0.rc.2 + L + + before { allow(Bundler::SharedHelpers).to receive(:find_gemfile).and_return(bundled_app("gems.rb")) } + subject { described_class.new(lockfile_contents) } + + it "parses the override remote" do + gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) } + expect(gem_source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end + + it "preserves the primary remote" do + gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) } + expect(gem_source.remotes.map(&:to_s)).to include("https://rubygems.org/") + end + end + + context "when a GEM source has multiple override remotes" do + let(:lockfile_contents) { <<~L } + GEM + remote: https://rubygems.org/ + override: https://first.example.com/ + override: https://second.example.com/ + specs: + rake (10.3.2) + + PLATFORMS + ruby + + DEPENDENCIES + rake + + BUNDLED WITH + 1.12.0.rc.2 + L + + before { allow(Bundler::SharedHelpers).to receive(:find_gemfile).and_return(bundled_app("gems.rb")) } + subject { described_class.new(lockfile_contents) } + + it "parses multiple override remotes" do + gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) } + expect(gem_source.override_remotes.size).to eq 2 + end + end + context "when the checksum is urlsafe base64 encoded" do let(:lockfile_contents) do super().sub( diff --git a/spec/bundler/source/rubygems_spec.rb b/spec/bundler/source/rubygems_spec.rb index dde4e4ed4769..890928cceb9d 100644 --- a/spec/bundler/source/rubygems_spec.rb +++ b/spec/bundler/source/rubygems_spec.rb @@ -31,6 +31,34 @@ end end + describe "#add_override_remote" do + it "normalizes and stores the URI" do + subject.add_override_remote("https://build-farm.example.com") + expect(subject.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end + + it "prepends new override remotes" do + subject.add_override_remote("https://first.example.com") + subject.add_override_remote("https://second.example.com") + expect(subject.override_remotes).to eq [ + Gem::URI("https://second.example.com/"), + Gem::URI("https://first.example.com/"), + ] + end + + it "does not add duplicates" do + subject.add_override_remote("https://build-farm.example.com") + subject.add_override_remote("https://build-farm.example.com") + expect(subject.override_remotes.size).to eq 1 + end + + context "when the source is an HTTP(s) URI with no host" do + it "raises error" do + expect { subject.add_override_remote("https:build-farm.example.com") }.to raise_error(ArgumentError) + end + end + end + describe "#no_remotes?" do context "when no remote provided" do it "returns a truthy value" do @@ -45,6 +73,191 @@ end end + describe "override remotes in initialize" do + it "stores override remotes from options" do + source = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end + + it "handles multiple override remotes" do + source = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://first.example.com", "https://second.example.com"] + ) + expect(source.override_remotes.size).to eq 2 + end + + it "defaults to empty override remotes" do + source = described_class.new("remotes" => ["https://rubygems.org"]) + expect(source.override_remotes).to eq [] + end + end + + describe "#eql?" do + it "considers sources with different override remotes as not equal" do + without_overrides = described_class.new("remotes" => ["https://rubygems.org"]) + with_overrides = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(without_overrides).not_to eql(with_overrides) + end + + it "considers sources with same remotes and override remotes as equal" do + source_a = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + source_b = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(source_a).to eql(source_b) + end + end + + describe "#hash" do + it "differs for sources with different override remotes" do + without_overrides = described_class.new("remotes" => ["https://rubygems.org"]) + with_overrides = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(without_overrides.hash).not_to eq(with_overrides.hash) + end + end + + describe "#to_lock" do + it "includes override lines" do + source = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expected = <<~L + GEM + remote: https://rubygems.org/ + override: https://build-farm.example.com/ + specs: + L + expect(source.to_lock).to eq(expected) + end + + it "includes multiple override lines" do + source = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://first.example.com", "https://second.example.com"] + ) + lock = source.to_lock + expect(lock).to include("override: https://first.example.com/") + expect(lock).to include("override: https://second.example.com/") + end + + it "omits override lines when no override remotes" do + source = described_class.new("remotes" => ["https://rubygems.org"]) + expect(source.to_lock).not_to include("override:") + end + end + + describe ".from_lock" do + it "restores override remotes from lockfile options" do + source = described_class.from_lock( + "remote" => "https://rubygems.org/", + "override" => "https://build-farm.example.com/" + ) + expect(source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end + + it "restores multiple override remotes from lockfile options" do + source = described_class.from_lock( + "remote" => "https://rubygems.org/", + "override" => ["https://first.example.com/", "https://second.example.com/"] + ) + expect(source.override_remotes.size).to eq 2 + end + + it "handles missing override key" do + source = described_class.from_lock("remote" => "https://rubygems.org/") + expect(source.override_remotes).to eq [] + end + end + + describe "to_lock/from_lock round-trip" do + it "preserves override remotes" do + original = described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + + # Simulate lockfile parser: parse the to_lock output + lock_output = original.to_lock + opts = {} + lock_output.each_line do |line| + next unless line =~ /^\s+([a-z]+): (.*)$/i + key = $1 + value = $2 + if opts[key] + opts[key] = Array(opts[key]) + opts[key] << value + else + opts[key] = value + end + end + + restored = described_class.from_lock(opts) + expect(restored.remotes.map(&:to_s)).to eq(original.remotes.map(&:to_s)) + expect(restored.override_remotes.map(&:to_s)).to eq(original.override_remotes.map(&:to_s)) + end + end + + describe "#fetch_override_specs" do + let(:source) do + described_class.new( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + end + + let(:override_uri) { Gem::URI("https://build-farm.example.com/") } + + def stub_override_fetcher(specs) + override_index = Bundler::Index.new + Array(specs).each {|s| override_index << s } + fetcher = double(Bundler::Fetcher, uri: override_uri) + allow(fetcher).to receive(:specs_with_retry).and_return(override_index) + allow(source).to receive(:override_fetchers).and_return([fetcher]) + allow(source).to receive(:dependency_names).and_return([]) + end + + it "raises GemfileError when override provides a gem absent from primary" do + override_spec = Gem::Specification.new("missing-from-primary", "1.0.0") + stub_override_fetcher(override_spec) + + primary_idx = Bundler::Index.new + + expect do + source.send(:fetch_override_specs, primary_idx) + end.to raise_error(Bundler::GemfileError, /missing-from-primary/) + end + + it "skips override specs whose version does not match the primary source" do + primary_spec = Gem::Specification.new("json", "2.7.0") + override_spec = Gem::Specification.new("json", "2.7.1") + stub_override_fetcher(override_spec) + + primary_idx = Bundler::Index.new + primary_idx << primary_spec + + expect do + source.send(:fetch_override_specs, primary_idx) + end.not_to raise_error + + expect(primary_idx.search(["json", "2.7.1"])).to be_empty + end + end + describe "log debug information" do it "log the time spent downloading and installing a gem" do build_repo2 do diff --git a/spec/bundler/source_list_spec.rb b/spec/bundler/source_list_spec.rb index 6e0be6c92fcc..0aa69590f79d 100644 --- a/spec/bundler/source_list_spec.rb +++ b/spec/bundler/source_list_spec.rb @@ -129,6 +129,38 @@ Gem::URI("https://rubygems.org/"), ] end + + it "adds override remotes to the global source" do + source_list.add_global_rubygems_remote( + "https://rubygems.org", + overrides: ["https://build-farm.example.com"] + ) + expect(returned_source.override_remotes).to eq [ + Gem::URI("https://build-farm.example.com/"), + ] + end + + it "adds multiple override remotes to the global source" do + source_list.add_global_rubygems_remote( + "https://rubygems.org", + overrides: ["https://first.example.com", "https://second.example.com"] + ) + expect(returned_source.override_remotes.size).to eq 2 + end + + it "does not add override remotes when none are provided" do + expect(returned_source.override_remotes).to eq [] + end + end + + describe "#add_rubygems_source with overrides" do + it "passes override remotes to the new source" do + source = source_list.add_rubygems_source( + "remotes" => ["https://rubygems.org"], + "overrides" => ["https://build-farm.example.com"] + ) + expect(source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")] + end end describe "#add_plugin_source" do