diff --git a/Gemfile.lock b/Gemfile.lock index 1f6347f9..aa0f6323 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -5,12 +5,11 @@ PATH activesupport (>= 5.0, < 7) addressable (~> 2.6) algoliasearch (~> 1.0) - concurrent-ruby (~> 1.1) + async-http (~> 0.54) fuzzy_match (~> 2.0.4) nap (~> 1.0) netrc (~> 0.11) public_suffix (~> 4.0) - typhoeus (~> 1.0) GEM remote: https://rubygems.org/ @@ -27,15 +26,31 @@ GEM httpclient (~> 2.8, >= 2.8.3) json (>= 1.5.1) ast (2.4.1) + async (1.28.9) + console (~> 1.10) + nio4r (~> 2.3) + timers (~> 4.1) + async-http (0.54.1) + async (~> 1.25) + async-io (~> 1.28) + async-pool (~> 0.2) + protocol-http (~> 0.21.0) + protocol-http1 (~> 0.13.0) + protocol-http2 (~> 0.14.0) + async-io (1.30.2) + async (~> 1.14) + async-pool (0.3.5) + async (~> 1.25) awesome_print (1.8.0) bacon (1.2.0) coderay (1.1.3) concurrent-ruby (1.1.7) + console (1.10.2) + fiber-local crack (0.4.5) rexml - ethon (0.12.0) - ffi (>= 1.3.0) ffi (1.14.2) + fiber-local (1.0.0) fuzzy_match (2.0.4) hashdiff (1.0.1) httpclient (2.8.3) @@ -58,12 +73,20 @@ GEM mocha (>= 0.13.0) nap (1.1.0) netrc (0.11.0) + nio4r (2.5.7) notify (0.5.2) parallel (1.20.1) parser (3.0.0.0) ast (~> 2.4.1) prettybacon (0.0.2) bacon (~> 1.2) + protocol-hpack (1.4.2) + protocol-http (0.21.0) + protocol-http1 (0.13.2) + protocol-http (~> 0.19) + protocol-http2 (0.14.2) + protocol-hpack (~> 1.4) + protocol-http (~> 0.18) pry (0.13.1) coderay (~> 1.1) method_source (~> 1.0) @@ -92,13 +115,12 @@ GEM rubocop (>= 0.90.0, < 2.0) rubocop-ast (>= 0.4.0) ruby-progressbar (1.11.0) - typhoeus (1.4.0) - ethon (>= 0.9.0) + timers (4.3.3) tzinfo (2.0.4) concurrent-ruby (~> 1.0) unicode-display_width (2.0.0) vcr (6.0.0) - webmock (3.11.1) + webmock (3.12.2) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) @@ -127,4 +149,4 @@ DEPENDENCIES webrick (~> 1.7.0) BUNDLED WITH - 2.2.3 + 2.2.8 diff --git a/cocoapods-core.gemspec b/cocoapods-core.gemspec index 21de857c..39e86d19 100644 --- a/cocoapods-core.gemspec +++ b/cocoapods-core.gemspec @@ -20,11 +20,10 @@ Gem::Specification.new do |s| s.require_paths = %w{ lib } s.add_runtime_dependency 'activesupport', '>= 5.0', '< 7' + s.add_runtime_dependency 'async-http', '~> 0.54' s.add_runtime_dependency 'nap', '~> 1.0' s.add_runtime_dependency 'fuzzy_match', '~> 2.0.4' s.add_runtime_dependency 'algoliasearch', '~> 1.0' - s.add_runtime_dependency 'concurrent-ruby', '~> 1.1' - s.add_runtime_dependency 'typhoeus', '~> 1.0' s.add_runtime_dependency 'netrc', '~> 0.11' s.add_runtime_dependency 'addressable', '~> 2.6' s.add_runtime_dependency 'public_suffix', '~> 4.0' diff --git a/lib/cocoapods-core/cdn_source.rb b/lib/cocoapods-core/cdn_source.rb index 40b578ec..dc45b3f0 100644 --- a/lib/cocoapods-core/cdn_source.rb +++ b/lib/cocoapods-core/cdn_source.rb @@ -1,20 +1,21 @@ require 'cocoapods-core/source' require 'rest' -require 'concurrent' require 'netrc' +require 'base64' +require 'zlib' +require 'async' +require 'async/barrier' +require 'async/http' +require 'async/http/internet' require 'addressable' module Pod # Subclass of Pod::Source to provide support for CDN-based Specs repositories # class CDNSource < Source - include Concurrent - + FORCE_HTTP2 = true MAX_NUMBER_OF_RETRIES = (ENV['COCOAPODS_CDN_MAX_NUMBER_OF_RETRIES'] || 5).to_i - # Single thread executor for all network activity. - HYDRA_EXECUTOR = Concurrent::SingleThreadExecutor.new - - private_constant :HYDRA_EXECUTOR + REQUEST_TIMEOUT = (ENV['COCOAPODS_CDN_REQUEST_TIMEOUT'] || 10).to_i # @param [String] repo The name of the repository # @@ -30,6 +31,20 @@ def initialize(repo) super(repo) end + # @return [Async::HTTP::Internet] The async HTTP client. + # + def http_client + @http_client ||= + begin + options = { + :retries => 0, + } + options[:protocol] = Async::HTTP::Protocol::HTTP2 if FORCE_HTTP2 + + Async::HTTP::Internet.new(**options) + end + end + # @return [String] The URL of the source. # def url @@ -60,13 +75,15 @@ def preheat_existing_files files_to_update = files_definitely_to_update + deprecated_local_podspecs - ['deprecated_podspecs.txt'] debug "CDN: #{name} Going to update #{files_to_update.count} files" - concurrent_requests_catching_errors do + concurrent_requests_catching_errors do |task| # Queue all tasks first - loaders = files_to_update.map do |file| - download_file_async(file) + files_to_update.each do |file| + task.async do + download_file_async(file) + end end - # Block and wait for all to complete running on Hydra - Promises.zip_futures_on(HYDRA_EXECUTOR, *loaders).wait! + ensure + http_client.close end end @@ -118,33 +135,30 @@ def versions(name) return nil if @version_arrays_by_fragment_by_name[fragment][name].nil? - concurrent_requests_catching_errors do - loaders = [] - - @versions_by_name[name] ||= @version_arrays_by_fragment_by_name[fragment][name].map do |version| + concurrent_requests_catching_errors do |task| + @version_arrays_by_fragment_by_name[fragment][name].each do |version| # Optimization: ensure all the podspec files at least exist. The correct one will get refreshed # in #specification_path regardless. podspec_version_path_relative = Pathname.new(version).join("#{name}.podspec.json") unless pod_path_actual.join(podspec_version_path_relative).exist? # Queue all podspec download tasks first - loaders << download_file_async(pod_path_relative.join(podspec_version_path_relative).to_s) - end - - begin - Version.new(version) if version[0, 1] != '.' - rescue ArgumentError - raise Informative, 'An unexpected version directory ' \ - "`#{version}` was encountered for the " \ - "`#{pod_path_actual}` Pod in the `#{name}` repository." + task.async do + download_file_async(pod_path_relative.join(podspec_version_path_relative).to_s) + end end - end.compact.sort.reverse - - # Block and wait for all to complete running on Hydra - Promises.zip_futures_on(HYDRA_EXECUTOR, *loaders).wait! + end + ensure + http_client.close end - @versions_by_name[name] + @versions_by_name[name] ||= @version_arrays_by_fragment_by_name[fragment][name].map do |version| + Version.new(version) if version[0, 1] != '.' + rescue ArgumentError + raise Informative, 'An unexpected version directory ' \ + "`#{version}` was encountered for the " \ + "`#{pod_path_actual}` Pod in the `#{name}` repository." + end.compact.sort.reverse end # Returns the path of the specification with the given name and version. @@ -246,7 +260,7 @@ def search_by_name(query, full_text_search = false) # Check update dates for all existing files. # Does not download non-existing specs, since CDN-backed repo is updated live. # - # @param [Bool] show_output + # @param [Bool] _show_output # # @return [Array] Always returns empty array, as it cannot know # everything that actually changed. @@ -332,10 +346,13 @@ def relative_pod_path(pod_name) end def download_file(partial_url) - # Block the main thread waiting for Hydra to finish - # - # Used for single-file downloads - download_file_async(partial_url).wait! + Sync do + download_file_async(partial_url) + ensure + http_client.close + end + + partial_url end def download_file_async(partial_url) @@ -346,12 +363,12 @@ def download_file_async(partial_url) if file_okay if @startup_time < File.mtime(path) debug "CDN: #{name} Relative path: #{partial_url} modified during this run! Returning local" - return Promises.fulfilled_future(partial_url, HYDRA_EXECUTOR) + return end unless @check_existing_files_for_update debug "CDN: #{name} Relative path: #{partial_url} exists! Returning local because checking is only performed in repo update" - return Promises.fulfilled_future(partial_url, HYDRA_EXECUTOR) + return end end @@ -359,7 +376,7 @@ def download_file_async(partial_url) etag_path = path.sub_ext(path.extname + '.etag') - etag = File.read(etag_path) if file_okay && File.exist?(etag_path) + etag = file_okay && File.exist?(etag_path) ? File.read(etag_path) : nil debug "CDN: #{name} Relative path: #{partial_url}, has ETag? #{etag}" unless etag.nil? download_and_save_with_retries_async(partial_url, file_remote_url, etag) @@ -369,61 +386,105 @@ def download_and_save_with_retries_async(partial_url, file_remote_url, etag, ret path = repo + partial_url etag_path = path.sub_ext(path.extname + '.etag') - download_task = download_typhoeus_impl_async(file_remote_url, etag).then do |response| - case response.response_code - when 301, 302 - redirect_location = response.headers['location'] - debug "CDN: #{name} Redirecting from #{file_remote_url} to #{redirect_location}" - download_and_save_with_retries_async(partial_url, redirect_location, etag) - when 304 - debug "CDN: #{name} Relative path not modified: #{partial_url}" - # We need to update the file modification date, as it is later used for freshness - # optimization. See #initialize for more information. - FileUtils.touch path - partial_url - when 200 - File.open(path, 'w') { |f| f.write(response.response_body.force_encoding('UTF-8')) } - - etag_new = response.headers['etag'] unless response.headers.nil? - debug "CDN: #{name} Relative path downloaded: #{partial_url}, save ETag: #{etag_new}" - File.open(etag_path, 'w') { |f| f.write(etag_new) } unless etag_new.nil? - partial_url - when 404 - debug "CDN: #{name} Relative path couldn't be downloaded: #{partial_url} Response: #{response.response_code}" - nil - when 502, 503, 504 - # Retryable HTTP errors, usually related to server overloading - if retries <= 1 - raise Informative, "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.response_code} #{response.response_body}" - else - debug "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.response_code} #{response.response_body}, retries: #{retries - 1}" - exponential_backoff_async(retries).then do - download_and_save_with_retries_async(partial_url, file_remote_url, etag, retries - 1) - end + response = + begin + Async::Task.current.with_timeout(REQUEST_TIMEOUT) do + create_download_task(file_remote_url, etag) end - when 0 - # Non-HTTP errors, usually network layer + rescue Async::TimeoutError, Async::HTTP::Protocol::RequestFailed, SocketError, ::StandardError => e + message = + case e + when Async::TimeoutError + "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: Request timeout" + when SocketError + "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: Couldn't connect to server" + else + "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{e.message}" + end + if retries <= 1 - raise Informative, "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.return_message}" + raise Informative, message else - debug "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.return_message}, retries: #{retries - 1}" - exponential_backoff_async(retries).then do - download_and_save_with_retries_async(partial_url, file_remote_url, etag, retries - 1) + debug message + ", retries: #{retries - 1}" + sleep_async backoff_time(retries) + download_and_save_with_retries_async(partial_url, file_remote_url, etag, retries - 1) + return + end + end + + body = response.read + case response.status + when 301, 302 + redirect_location = response.headers['location'] + debug "CDN: #{name} Redirecting from #{file_remote_url} to #{redirect_location}" + download_and_save_with_retries_async(partial_url, redirect_location, etag) + when 304 + debug "CDN: #{name} Relative path not modified: #{partial_url}" + # We need to update the file modification date, as it is later used for freshness + # optimization. See #initialize for more information. + FileUtils.touch path + when 200 + File.open(path, 'w') do |f| + encoding = response.headers['content-encoding'].to_s + if encoding.present? + case encoding + when 'gzip' + body = Zlib::GzipReader.wrap(StringIO.new(body), &:read) + else + raise Informative, "CDN: #{name} URL couldn't be saved: #{file_remote_url} Content encoding: #{response.headers['content-encoding']}" end end + + f.write(body&.force_encoding('UTF-8')) + end + + etag_new = response.headers['etag'] + debug "CDN: #{name} Relative path downloaded: #{partial_url}, save ETag: #{etag_new}" + File.open(etag_path, 'w') { |f| f.write(etag_new) } unless etag_new.nil? + when 404 + debug "CDN: #{name} Relative path couldn't be downloaded: #{partial_url} Response: #{response.status}" + nil + when 502, 503, 504 + # Retryable HTTP errors, usually related to server overloading + message = "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.status} #{body}" + if retries <= 1 + raise Informative, message else - raise Informative, "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.response_code} #{response.response_body}" + debug message + ", retries: #{retries - 1}" + sleep_async backoff_time(retries) + download_and_save_with_retries_async(partial_url, file_remote_url, etag, retries - 1) + end + else + raise Informative, "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.status} #{body}" + end + end + + def create_download_task(file_remote_url, etag = nil) + headers = [ + %w[Accept-Encoding gzip], + ] + + unless etag.nil? + headers << ['If-None-Match', etag] + end + + begin + netrc_info = Netrc.read + netrc_host = URI.parse(file_remote_url).host + credentials = netrc_info[netrc_host] + if credentials + user, pass = credentials + headers << ['Authorization', Protocol::HTTP::Header::Authorization.basic(user, pass)] end + rescue Netrc::Error => e + raise Informative, "CDN: #{e.message}" end - # Calling `Future#run` flattens the chained futures created by retries or redirects - # - # Does not, in fact, run the task - that is already happening in Hydra at this point - download_task.run + http_client.get file_remote_url, headers end - def exponential_backoff_async(retries) - sleep_async(backoff_time(retries)) + def sleep_async(seconds) + Async::Task.current.sleep seconds end def backoff_time(retries) @@ -431,42 +492,6 @@ def backoff_time(retries) 4 * 2**current_retry end - def sleep_async(seconds) - # Async sleep to avoid blocking either the main or the Hydra thread - Promises.schedule_on(HYDRA_EXECUTOR, seconds) - end - - def download_typhoeus_impl_async(file_remote_url, etag) - require 'typhoeus' - - # Create a prefereably HTTP/2 request - the protocol is ultimately responsible for picking - # the maximum supported protocol - # When debugging with proxy, use the following extra options: - # :proxy => 'http://localhost:8888', - # :ssl_verifypeer => false, - # :ssl_verifyhost => 0, - request = Typhoeus::Request.new( - file_remote_url, - :method => :get, - :http_version => :httpv2_0, - :timeout => 10, - :connecttimeout => 10, - :accept_encoding => 'gzip', - :netrc => :optional, - :netrc_file => Netrc.default_path, - :headers => etag.nil? ? {} : { 'If-None-Match' => etag }, - ) - - future = Promises.resolvable_future_on(HYDRA_EXECUTOR) - queue_request(request) - request.on_complete do |response| - future.fulfill(response) - end - - # This `Future` should never reject, network errors are exposed on `Typhoeus::Response` - future - end - def debug(message) if defined?(Pod::UI) Pod::UI.message(message) @@ -476,26 +501,26 @@ def debug(message) end def concurrent_requests_catching_errors - yield - rescue MultipleErrors => e - # aggregated error message from `Concurrent` - errors = e.errors - raise Informative, "CDN: #{name} Repo update failed - #{e.errors.size} error(s):\n#{errors.join("\n")}" - end - - def queue_request(request) - @hydra ||= Typhoeus::Hydra.new - - # Queue the request into the Hydra (libcurl reactor). - @hydra.queue(request) - - # Cycle the reactor on a separate thread - # - # The way it works is that if more requests are queued while Hydra is in the `#run` - # method, it will keep executing them - # - # The upcoming calls to `#run` will simply run empty. - HYDRA_EXECUTOR.post(@hydra, &:run) + errors = [] + results = [] + + Sync do |task| + barrier = Async::Barrier.new(:parent => task) + + yield barrier + + barrier.tasks.each do |child| + results << child.result + rescue ::StandardError => e + errors << e + end + end + + if errors.any? + raise Informative, "CDN: #{name} Repo update failed - #{errors.size} error(s):\n#{errors.join("\n")}" + end + + results end end end diff --git a/spec/cdn_source_spec.rb b/spec/cdn_source_spec.rb index af7fe4da..225924e1 100644 --- a/spec/cdn_source_spec.rb +++ b/spec/cdn_source_spec.rb @@ -1,7 +1,5 @@ require 'fileutils' require 'algoliasearch' -require 'concurrent' -require 'typhoeus' require File.expand_path('../spec_helper', __FILE__) module Mocha @@ -49,29 +47,6 @@ def print_dir(tag) STDERR.puts Pathname.glob(@path.join('*')).sort.join("\n") end - def fulfilled_future(result) - Concurrent::Promises.fulfilled_future(result) - end - - def resolved_event - Concurrent::Promises.resolved_event - end - - def typhoeus_http_response_future(code, headers = {}, body = '') - fulfilled_future(Typhoeus::Response.new( - :response_code => code, - :headers => headers, - :response_body => body, - )) - end - - def typhoeus_non_http_response_future(code) - fulfilled_future(Typhoeus::Response.new( - :response_code => 0, - :return_code => code, - )) - end - @remote_dir = fixture('mock_cdn_repo_remote') @path = fixture('spec-repos/test_cdn_repo_local') @@ -79,9 +54,19 @@ def typhoeus_non_http_response_future(code) save_url('http://localhost:4321/') @source = CDNSource.new(@path) + @source.expects(:sleep_async).never + + @save_log_level = Async.logger.level + + # silence "unawaited task error" false positives to reduce log noise + # see https://github.com/socketry/async/issues/91 for more discussion + # uncomment for debug purposes + Async.logger.level = :fatal end after do + WebMock.reset! + Async.logger.level = @save_log_level cleanup end @@ -158,7 +143,7 @@ def typhoeus_non_http_response_future(code) @source = CDNSource.new(@path) netrc_file = temporary_directory + '.netrc' - File.open(netrc_file, 'w') { |f| f.write("machine localhost\nlogin user1\npassword xxx\n") } + File.open(netrc_file, 'w', 0o600) { |f| f.write("machine localhost\nlogin user1\npassword xxx\n") } ENV['NETRC'] = temporary_directory.to_s auth = nil @@ -174,15 +159,13 @@ def typhoeus_non_http_response_future(code) relative_path = 'all_pods_versions_2_0_9.txt' original_url = 'http://localhost:4321/' + relative_path redirect_url = 'http://localhost:4321/redirected/' + relative_path - @source.expects(:download_typhoeus_impl_async). - with_url(original_url). - returns(typhoeus_http_response_future(301, 'location' => redirect_url)) - @source.expects(:download_typhoeus_impl_async). - with_url(redirect_url). - returns(typhoeus_http_response_future(200, {}, 'BeaconKit/1.0.0')) - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.0/BeaconKit.podspec.json'). - returns(typhoeus_http_response_future(200, {}, '')) + + WebMock.stub_request(:get, original_url). + to_return(:status => 301, :headers => { 'location' => redirect_url }) + WebMock.stub_request(:get, redirect_url). + to_return(:status => 200, :headers => {}, :body => 'BeaconKit/1.0.0') + WebMock.stub_request(:get, 'http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.0/BeaconKit.podspec.json'). + to_return(:status => 200, :headers => {}, :body => '') @source.expects(:debug).with { |cmd| cmd.include? "CDN: #{@source.name} Relative path downloaded: all_pods_versions_2_0_9.txt, save ETag:" } @source.expects(:debug).with("CDN: #{@source.name} Redirecting from #{original_url} to #{redirect_url}") @@ -190,47 +173,23 @@ def typhoeus_non_http_response_future(code) @source.versions('BeaconKit').map(&:to_s).should == %w(1.0.0) end - it 'handles responses with no headers' do - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/all_pods_versions_2_0_9.txt'). - returns(typhoeus_http_response_future(200, nil, 'BeaconKit/1.0.0')) - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.0/BeaconKit.podspec.json'). - returns(typhoeus_http_response_future(200, nil, '{}')) - should.not.raise do - @source.versions('BeaconKit') - end - end - - it 'forces UTF-8 encoding for the body' do - mock_json_body = mock - mock_json_body.expects(:force_encoding).with('UTF-8').at_least_once - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/all_pods_versions_2_0_9.txt'). - returns(typhoeus_http_response_future(200, nil, 'BeaconKit/1.0.0')) - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.0/BeaconKit.podspec.json'). - returns(typhoeus_http_response_future(200, {}, mock_json_body)) - should.not.raise do - @source.versions('BeaconKit') - end - end - it 'raises if unexpected HTTP error' do - @source.expects(:download_typhoeus_impl_async). - returns(typhoeus_http_response_future(500)) + WebMock.stub_request(:get, 'http://localhost:4321/all_pods_versions_2_0_9.txt'). + to_return(:status => 500, :headers => {}, :body => '') + should.raise Informative do @source.versions('BeaconKit') end.message. - should.include "CDN: #{@source.name} URL couldn\'t be downloaded: #{@url}all_pods_versions_2_0_9.txt Response: 500" + should.include "CDN: #{@source.name} URL couldn't be downloaded: #{@url}all_pods_versions_2_0_9.txt Response: 500" end it 'raises if unexpected non-HTTP error' do - @source.expects(:download_typhoeus_impl_async). - at_least_once. - returns(typhoeus_non_http_response_future(:couldnt_connect)) + WebMock.stub_request(:get, 'http://localhost:4321/all_pods_versions_2_0_9.txt'). + to_raise(SocketError) - @source.expects(:sleep_async).at_least_once.with(anything).returns(resolved_event) + [4, 8, 16, 32].each do |seconds| + @source.expects(:sleep_async).with(seconds) + end should.raise Informative do @source.versions('BeaconKit') @@ -239,38 +198,33 @@ def typhoeus_non_http_response_future(code) end it 'retries after unexpected HTTP error' do - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/all_pods_versions_2_0_9.txt'). - at_most(5). - returns(typhoeus_http_response_future(503)). - returns(typhoeus_http_response_future(503)). - returns(typhoeus_http_response_future(503)). - returns(typhoeus_http_response_future(503)). - then. - returns(typhoeus_http_response_future(200, {}, 'BeaconKit/1.0.0')) - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.0/BeaconKit.podspec.json'). - returns(typhoeus_http_response_future(200, {}, '')) + WebMock.stub_request(:get, 'http://localhost:4321/all_pods_versions_2_0_9.txt'). + to_return(:status => 503, :headers => {}, :body => ''). + to_return(:status => 503, :headers => {}, :body => ''). + to_return(:status => 503, :headers => {}, :body => ''). + to_return(:status => 503, :headers => {}, :body => ''). + to_return(:status => 200, :headers => {}, :body => 'BeaconKit/1.0.0') + + WebMock.stub_request(:get, 'http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.0/BeaconKit.podspec.json'). + to_return(:status => 200, :headers => {}, :body => '') [4, 8, 16, 32].each do |seconds| - @source.expects(:sleep_async).with(seconds).returns(resolved_event) + @source.expects(:sleep_async).with(seconds) end @source.versions('BeaconKit').map(&:to_s).should == %w(1.0.0) end it 'fails after unexpected HTTP error retries are exhausted' do - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/all_pods_versions_2_0_9.txt'). - at_most(5). - returns(typhoeus_http_response_future(503)). - returns(typhoeus_http_response_future(503)). - returns(typhoeus_http_response_future(503)). - returns(typhoeus_http_response_future(503)). - returns(typhoeus_http_response_future(503)) + WebMock.stub_request(:get, 'http://localhost:4321/all_pods_versions_2_0_9.txt'). + to_return(:status => 503, :headers => {}, :body => ''). + to_return(:status => 503, :headers => {}, :body => ''). + to_return(:status => 503, :headers => {}, :body => ''). + to_return(:status => 503, :headers => {}, :body => ''). + to_return(:status => 503, :headers => {}, :body => '') [4, 8, 16, 32].each do |seconds| - @source.expects(:sleep_async).with(seconds).returns(resolved_event) + @source.expects(:sleep_async).with(seconds) end should.raise Informative do @@ -279,38 +233,33 @@ def typhoeus_non_http_response_future(code) end it 'retries after unexpected non-HTTP error' do - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/all_pods_versions_2_0_9.txt'). - at_most(5). - returns(typhoeus_non_http_response_future(:couldnt_connect)). - returns(typhoeus_non_http_response_future(:couldnt_connect)). - returns(typhoeus_non_http_response_future(:couldnt_connect)). - returns(typhoeus_non_http_response_future(:couldnt_connect)). - then. - returns(typhoeus_http_response_future(200, {}, 'BeaconKit/1.0.0')) - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.0/BeaconKit.podspec.json'). - returns(typhoeus_http_response_future(200, {}, '')) + WebMock.stub_request(:get, 'http://localhost:4321/all_pods_versions_2_0_9.txt'). + to_raise(SocketError). + to_raise(SocketError). + to_raise(SocketError). + to_raise(SocketError). + to_return(:status => 200, :headers => {}, :body => 'BeaconKit/1.0.0') + + WebMock.stub_request(:get, 'http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.0/BeaconKit.podspec.json'). + to_return(:status => 200, :headers => {}, :body => '') [4, 8, 16, 32].each do |seconds| - @source.expects(:sleep_async).with(seconds).returns(resolved_event) + @source.expects(:sleep_async).with(seconds) end @source.versions('BeaconKit').map(&:to_s).should == %w(1.0.0) end it 'fails after unexpected non-HTTP error retries are exhausted' do - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/all_pods_versions_2_0_9.txt'). - at_most(5). - returns(typhoeus_non_http_response_future(:couldnt_connect)). - returns(typhoeus_non_http_response_future(:couldnt_connect)). - returns(typhoeus_non_http_response_future(:couldnt_connect)). - returns(typhoeus_non_http_response_future(:couldnt_connect)). - returns(typhoeus_non_http_response_future(:couldnt_connect)) + WebMock.stub_request(:get, 'http://localhost:4321/all_pods_versions_2_0_9.txt'). + to_raise(SocketError). + to_raise(SocketError). + to_raise(SocketError). + to_raise(SocketError). + to_raise(SocketError) [4, 8, 16, 32].each do |seconds| - @source.expects(:sleep_async).with(seconds).returns(resolved_event) + @source.expects(:sleep_async).with(seconds) end should.raise Informative do @@ -318,16 +267,14 @@ def typhoeus_non_http_response_future(code) end.message.should.include "CDN: #{@source.name} URL couldn't be downloaded: http://localhost:4321/all_pods_versions_2_0_9.txt Response: Couldn't connect to server" end - it 'raises cumulative error when more than one Future rejects' do - @source.expects(:download_typhoeus_impl_async). - with_url('http://localhost:4321/all_pods_versions_2_0_9.txt'). - returns(typhoeus_http_response_future(200, {}, 'BeaconKit/1.0.0/1.0.1/1.0.2/1.0.3/1.0.4/1.0.5')) - versions = %w(0 1 2 3 4 5) - messages = versions.map do |index| - @source.expects(:download_typhoeus_impl_async). - at_least_once. - with_url("http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.#{index}/BeaconKit.podspec.json"). - returns(typhoeus_http_response_future(500, {}, 'Some error')) + it 'raises cumulative error when concurrent requests have errors' do + WebMock.stub_request(:get, 'http://localhost:4321/all_pods_versions_2_0_9.txt'). + to_return(:status => 200, :headers => {}, :body => 'BeaconKit/1.0.0/1.0.1/1.0.2/1.0.3/1.0.4/1.0.5') + + messages = %w(0 1 2 3 4 5).map do |index| + WebMock.stub_request(:get, "http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.#{index}/BeaconKit.podspec.json"). + to_return(:status => 500, :headers => {}, :body => 'Some error') + "CDN: #{@source.name} URL couldn't be downloaded: #{@url}Specs/2/0/9/BeaconKit/1.0.#{index}/BeaconKit.podspec.json Response: 500 Some error" end @@ -336,6 +283,24 @@ def typhoeus_non_http_response_future(code) end.message.should.include "CDN: #{@source.name} Repo update failed - 6 error(s):\n" + messages.join("\n") end + it 'raises cumulative error only for errored requests' do + WebMock.stub_request(:get, 'http://localhost:4321/all_pods_versions_2_0_9.txt'). + to_return(:status => 200, :headers => {}, :body => 'BeaconKit/1.0.0/1.0.1/1.0.2/1.0.3/1.0.4/1.0.5') + WebMock.stub_request(:get, 'http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.4/BeaconKit.podspec.json'). + to_return(:status => 200, :headers => {}, :body => 'BeaconKit/1.0.0/1.0.1/1.0.2/1.0.3/1.0.4/1.0.5') + + messages = %w(0 1 2 3 5).map do |index| + WebMock.stub_request(:get, "http://localhost:4321/Specs/2/0/9/BeaconKit/1.0.#{index}/BeaconKit.podspec.json"). + to_return(:status => 500, :headers => {}, :body => 'Some error') + + "CDN: #{@source.name} URL couldn't be downloaded: #{@url}Specs/2/0/9/BeaconKit/1.0.#{index}/BeaconKit.podspec.json Response: 500 Some error" + end + + should.raise Informative do + @source.versions('BeaconKit') + end.message.should.include "CDN: #{@source.name} Repo update failed - 5 error(s):\n" + messages.join("\n") + end + it 'returns cached versions for a Pod' do pod_path_children = %w(1.0.5 1.0.4 1.0.3 1.0.2 1.0.1 1.0.0) @source.versions('BeaconKit').map(&:to_s).should == pod_path_children @@ -502,9 +467,10 @@ def typhoeus_non_http_response_future(code) describe '#update' do it 'returns empty array' do - File.open(@path.join('deprecated_podspecs.txt'), 'w') { |f| } + File.open(@path.join('deprecated_podspecs.txt'), 'w') {} + CDNSource.any_instance.expects(:download_file).with('deprecated_podspecs.txt').returns('deprecated_podspecs.txt') - CDNSource.any_instance.expects(:download_file_async).with('CocoaPods-version.yml').returns(fulfilled_future('CocoaPods-version.yml')) + CDNSource.any_instance.expects(:download_file_async).with('CocoaPods-version.yml') @source.update(true).should == [] end end @@ -542,8 +508,8 @@ def typhoeus_non_http_response_future(code) it 'refreshes all index files' do File.open(@path.join('deprecated_podspecs.txt'), 'w') { |f| } @source.expects(:download_file).with('deprecated_podspecs.txt').returns('deprecated_podspecs.txt') - @source.expects(:download_file_async).with('CocoaPods-version.yml').returns(fulfilled_future('CocoaPods-version.yml')) - @source.expects(:download_file_async).with('all_pods_versions_2_0_9.txt').returns(fulfilled_future('all_pods_versions_2_0_9.txt')) + @source.expects(:download_file_async).with('CocoaPods-version.yml') + @source.expects(:download_file_async).with('all_pods_versions_2_0_9.txt') @source.update(true) end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ccfe4254..72ed77c1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -54,6 +54,13 @@ def copy_fixture_to_pod(name, pod) Thread.current.exit end +# Force CDNSource uses HTTP1, because WebMock doesn't support HTTP 2 yet +#--------------------------------------# + +silence_warnings do + Pod::CDNSource::FORCE_HTTP2 = false +end + # Silence the output #--------------------------------------#