Allow declaring override remotes on a RubyGems source

bundler/lib/bundler/dsl.rb

@@ -116,6 +116,7 @@ def source(source, *args, &blk)
       options = args.last.is_a?(Hash) ? args.pop.dup : {}
       options = normalize_hash(options)
       source = normalize_source(source)
+      overrides = options["overrides"]
 
       if options.key?("type")
         options["type"] = options["type"].to_s
@@ -130,9 +131,12 @@ def source(source, *args, &blk)
         source_opts = options.merge("uri" => source)
         with_source(@sources.add_plugin_source(options["type"], source_opts), &blk)
       elsif block_given?
-        with_source(@sources.add_rubygems_source("remotes" => source), &blk)
+        with_source(@sources.add_rubygems_source(
+                      "remotes" => source,
+                      "overrides" => overrides
+                    ), &blk)
       else
-        @sources.add_global_rubygems_remote(source)
+        @sources.add_global_rubygems_remote(source, overrides: overrides)
       end
     end
 

bundler/lib/bundler/man/gemfile.5.ronn

@@ -55,6 +55,33 @@ include the credentials in the Gemfile as part of the source URL.
 Credentials in the source URL will take precedence over credentials set using
 `config`.
 
+### OVERRIDES
+
+A `source` may declare one or more `overrides`. An override is a secondary
+RubyGems-compatible repository that supplies alternate builds (for example,
+prebuilt binaries) of gems that already exist in the primary source.
+
+    source "https://rubygems.org",
+           overrides: ["https://build-farm.example.com"]
+
+Overrides are consulted only for gems that are also present in the primary
+source — they cannot introduce new gems. If an override publishes a gem that
+the primary source does not contain in any version, Bundler raises an error,
+since this indicates the override and primary source are misconfigured.
+
+When an override publishes a spec whose name and version match one in the
+primary source, that spec is preferred for installation if it is compatible
+with the local platform. If no matching build is available, or fetching the
+override fails (authentication, SSL, network), Bundler falls back to the
+primary source.
+
+Overrides are recorded in `Gemfile.lock` under `override:` lines:
+
+    GEM
+      remote: https://rubygems.org/
+      override: https://build-farm.example.com/
+      specs:
+
 ## RUBY
 
 If your application requires a specific Ruby version or engine, specify your

bundler/lib/bundler/source/rubygems.rb

@@ -12,10 +12,12 @@ class Rubygems < Source
       REQUIRE_MUTEX = Mutex.new
 
       attr_accessor :remotes
+      attr_reader :override_remotes
 
       def initialize(options = {})
         @options = options
         @remotes = []
+        @override_remotes = []
         @dependency_names = []
         @allow_remote = false
         @allow_cached = false
@@ -26,8 +28,12 @@ def initialize(options = {})
         @gem_installers_mutex = Mutex.new
 
         Array(options["remotes"]).reverse_each {|r| add_remote(r) }
+        Array(options["overrides"]).reverse_each {|override| add_override_remote(override) }
 
-        @lockfile_remotes = @remotes if options["from_lockfile"]
+        if options["from_lockfile"]
+          @lockfile_remotes = @remotes
+          @lockfile_override_remotes = @override_remotes
+        end
       end
 
       def caches
@@ -73,11 +79,13 @@ def cached!
       end
 
       def hash
-        @remotes.hash
+        [@remotes, @override_remotes].hash
       end
 
       def eql?(other)
-        other.is_a?(Rubygems) && other.credless_remotes == credless_remotes
+        other.is_a?(Rubygems) &&
+          other.credless_remotes == credless_remotes &&
+          other.override_remotes == override_remotes
       end
 
       alias_method :==, :eql?
@@ -105,6 +113,7 @@ def options
 
       def self.from_lock(options)
         options["remotes"] = Array(options.delete("remote")).reverse
+        options["overrides"] = Array(options.delete("override")).reverse
         new(options.merge("from_lockfile" => true))
       end
 
@@ -113,6 +122,9 @@ def to_lock
         lockfile_remotes.reverse_each do |remote|
           out << "  remote: #{remote}\n"
         end
+        lockfile_override_remotes.reverse_each do |override|
+          out << "  override: #{override}\n"
+        end
         out << "  specs:\n"
       end
 
@@ -248,6 +260,11 @@ def add_remote(source)
         @remotes.unshift(uri) unless @remotes.include?(uri)
       end
 
+      def add_override_remote(source)
+        uri = normalize_uri(source)
+        @override_remotes.unshift(uri) unless @override_remotes.include?(uri)
+      end
+
       def spec_names
         if dependency_api_available?
           remote_specs.spec_names
@@ -275,6 +292,17 @@ def fetchers
         @fetchers ||= remote_fetchers.values.freeze
       end
 
+      def override_remote_fetchers
+        @override_remote_fetchers ||= @override_remotes.to_h do |uri|
+          remote = Source::Rubygems::Remote.new(uri)
+          [remote, Bundler::Fetcher.new(remote)]
+        end.freeze
+      end
+
+      def override_fetchers
+        @override_fetchers ||= override_remote_fetchers.values.freeze
+      end
+
       def double_check_for(unmet_dependency_names)
         return unless dependency_api_available?
 
@@ -324,6 +352,10 @@ def credless_remotes
         remotes.map(&method(:remove_auth))
       end
 
+      def credless_override_remotes
+        override_remotes.map(&method(:remove_auth))
+      end
+
       def cached_gem(spec)
         global_cache_path = download_cache_path(spec)
         caches << global_cache_path if global_cache_path
@@ -399,6 +431,8 @@ def remote_specs
           else
             fetch_names(fetchers, nil, idx)
           end
+
+          fetch_override_specs(idx) if @override_remotes.any? && @allow_remote
         end
       end
 
@@ -415,6 +449,51 @@ def fetch_names(fetchers, dependency_names, index)
         end
       end
 
+      # Merge spec from override remotes into +idx+, but only for gems already
+      # present in the parent source.
+      def fetch_override_specs(idx)
+        local_platform = Bundler.local_platform
+
+        override_fetchers.each do |fetcher|
+          filtered_uri = URICredentialsFilter.credential_filtered_uri(fetcher.uri)
+          begin
+            Bundler.ui.info "Fetching override gem metadata from #{filtered_uri}", Bundler.ui.debug?
+            override_index = fetcher.specs_with_retry(dependency_names, self)
+            Bundler.ui.info "" unless Bundler.ui.debug?
+
+            override_index.each do |spec|
+              primary_specs = idx.search(spec.name)
+              if primary_specs.empty?
+                raise GemfileError, "Override source #{filtered_uri} provides #{spec.name}, but no version of #{spec.name} exists in the primary source. Override sources may only supply alternate builds for gems already present in the primary source."
+              end
+
+              unless primary_specs.any? {|s| s.version == spec.version }
+                Bundler.ui.debug "Skipping #{spec.full_name} from override source #{filtered_uri} (no matching version in parent source)"
+                next
+              end
+
+              unless spec.installable_on_platform?(local_platform)
+                Bundler.ui.debug "Skipping #{spec.full_name} from override source #{filtered_uri} (platform #{spec.platform} not compatible with #{local_platform})"
+                next
+              end
+
+              idx << spec
+            end
+          rescue Bundler::Fetcher::AuthenticationRequiredError, Bundler::Fetcher::BadAuthenticationError, Bundler::Fetcher::AuthenticationForbiddenError => e
+            warn_override_failure(filtered_uri, "requires authentication", e)
+          rescue Bundler::Fetcher::CertificateFailureError, Bundler::Fetcher::SSLError => e
+            warn_override_failure(filtered_uri, "has SSL errors", e)
+          rescue Bundler::Fetcher::FallbackError, Bundler::HTTPError => e
+            warn_override_failure(filtered_uri, "is unreachable", e)
+          end
+        end
+      end
+
+      def warn_override_failure(filtered_uri, reason, error)
+        Bundler.ui.warn "Override source #{filtered_uri} #{reason}: #{error.message}. Falling back to source compilation."
+        Bundler.ui.debug "#{error.class}: #{error.message}"
+      end
+
       def fetch_gem_if_possible(spec, previous_spec = nil)
         if spec.remote
           fetch_gem(spec, previous_spec)
@@ -460,6 +539,15 @@ def lockfile_remotes
         @lockfile_remotes || credless_remotes
       end
 
+      def lockfile_override_remotes
+        @lockfile_override_remotes || credless_override_remotes
+      end
+
+      # Combined lookup for download_gem - includes both primary and override fetchers
+      def all_remote_fetchers
+        @all_remote_fetchers ||= remote_fetchers.merge(override_remote_fetchers).freeze
+      end
+
       # Checks if the requested spec exists in the global cache. If it does,
       # we copy it to the download path, and if it does not, we download it.
       #
@@ -475,7 +563,7 @@ def lockfile_remotes
       def download_gem(spec, download_cache_path, previous_spec = nil)
         uri = spec.remote.uri
         Bundler.ui.confirm("Fetching #{version_message(spec, previous_spec)}")
-        gem_remote_fetcher = remote_fetchers.fetch(spec.remote).gem_remote_fetcher
+        gem_remote_fetcher = all_remote_fetchers.fetch(spec.remote).gem_remote_fetcher
 
         Gem.time("Downloaded #{spec.name} in", 0, true) do
           Bundler.rubygems.download_gem(spec, uri, download_cache_path, gem_remote_fetcher)

bundler/lib/bundler/source_list.rb

@@ -59,8 +59,11 @@ def add_plugin_source(source, options = {})
       add_source_to_list Plugin.source(source).new(options), @plugin_sources
     end
 
-    def add_global_rubygems_remote(uri)
+    def add_global_rubygems_remote(uri, overrides: nil)
       global_rubygems_source.add_remote(uri)
+
+      overrides&.each {|override| global_rubygems_source.add_override_remote(override) }
+
       global_rubygems_source
     end
 

spec/bundler/lockfile_parser_spec.rb

@@ -164,6 +164,69 @@
       include_examples "parsing"
     end
 
+    context "when a GEM source has override remotes" do
+      let(:lockfile_contents) { <<~L }
+        GEM
+          remote: https://rubygems.org/
+          override: https://build-farm.example.com/
+          specs:
+            rake (10.3.2)
+
+        PLATFORMS
+          ruby
+
+        DEPENDENCIES
+          rake
+
+        CHECKSUMS
+          rake (10.3.2) sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8
+
+        BUNDLED WITH
+           1.12.0.rc.2
+      L
+
+      before { allow(Bundler::SharedHelpers).to receive(:find_gemfile).and_return(bundled_app("gems.rb")) }
+      subject { described_class.new(lockfile_contents) }
+
+      it "parses the override remote" do
+        gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) }
+        expect(gem_source.override_remotes).to eq [Gem::URI("https://build-farm.example.com/")]
+      end
+
+      it "preserves the primary remote" do
+        gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) }
+        expect(gem_source.remotes.map(&:to_s)).to include("https://rubygems.org/")
+      end
+    end
+
+    context "when a GEM source has multiple override remotes" do
+      let(:lockfile_contents) { <<~L }
+        GEM
+          remote: https://rubygems.org/
+          override: https://first.example.com/
+          override: https://second.example.com/
+          specs:
+            rake (10.3.2)
+
+        PLATFORMS
+          ruby
+
+        DEPENDENCIES
+          rake
+
+        BUNDLED WITH
+           1.12.0.rc.2
+      L
+
+      before { allow(Bundler::SharedHelpers).to receive(:find_gemfile).and_return(bundled_app("gems.rb")) }
+      subject { described_class.new(lockfile_contents) }
+
+      it "parses multiple override remotes" do
+        gem_source = subject.sources.find {|source| source.is_a?(Bundler::Source::Rubygems) }
+        expect(gem_source.override_remotes.size).to eq 2
+      end
+    end
+
     context "when the checksum is urlsafe base64 encoded" do
       let(:lockfile_contents) do
         super().sub(