From 86d3bf6a2aa65167a619b7b833cab5f1f2a6c015 Mon Sep 17 00:00:00 2001 From: Seth Madison Date: Sat, 7 Feb 2015 13:09:43 -0500 Subject: [PATCH 1/3] Add support for :file and :writedata options to easy These options take a file path as a string, and will write the body of the response directly to the passed path, instead of pulling it into the heap. When passed, these options set :writefunction to NULL, so that Curl knows to write directly to a file. Easy manages the file handle, and closes it automatically in on_complete. --- lib/ethon/curls/options.rb | 8 ++++++-- lib/ethon/easy.rb | 9 ++++++++- lib/ethon/easy/callbacks.rb | 13 ++++++++++--- lib/ethon/easy/files.rb | 18 ++++++++++++++++++ lib/ethon/libc.rb | 13 +++++++++++++ 5 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 lib/ethon/easy/files.rb diff --git a/lib/ethon/curls/options.rb b/lib/ethon/curls/options.rb index 1dac6f8c..e389cca3 100644 --- a/lib/ethon/curls/options.rb +++ b/lib/ethon/curls/options.rb @@ -57,6 +57,9 @@ def set_option(option, value, handle, type = :easy) s = value.to_s unless value.nil? value = FFI::MemoryPointer.new(:char, s.bytesize) value.put_bytes(0, s) + when :file + func=:ffipointer + # We expect that easy is passing us a FILE * when :string_escape_null func=:string value=Util.escape_zero_byte(value) unless value.nil? @@ -126,7 +129,8 @@ def set_option(option, value, handle, type = :easy) :ffipointer => :objectpoint, # FFI::Pointer :curl_slist => :objectpoint, :buffer => :objectpoint, # A memory buffer of size defined in the options - :dontuse_object => :objectpoint, # An object we don't support (e.g. FILE*) + :dontuse_object => :objectpoint, # An object we don't support + :file => :objectpoint, :cbdata => :objectpoint, :callback => :functionpoint, :debug_callback => :functionpoint, @@ -227,7 +231,7 @@ def #{type.to_s.downcase}_options(rt=nil) option :easy, :wildcardmatch, :bool, 197 ## CALLBACK OPTIONS option :easy, :writefunction, :callback, 11 - option :easy, :file, :cbdata, 1 + option :easy, :file, :file, 1 option_alias :easy, :file, :writedata option :easy, :readfunction, :callback, 12 option :easy, :infile, :cbdata, 9 diff --git a/lib/ethon/easy.rb b/lib/ethon/easy.rb index 03dedea3..0e8b7948 100644 --- a/lib/ethon/easy.rb +++ b/lib/ethon/easy.rb @@ -10,6 +10,7 @@ require 'ethon/easy/response_callbacks' require 'ethon/easy/debug_info' require 'ethon/easy/mirror' +require 'ethon/easy/files' module Ethon @@ -39,6 +40,7 @@ class Easy include Ethon::Easy::Http include Ethon::Easy::Operations include Ethon::Easy::ResponseCallbacks + include Ethon::Easy::Files # Returns the curl return code. # @@ -216,7 +218,7 @@ class Easy def initialize(options = {}) Curl.init set_attributes(options) - set_callbacks + set_callbacks(options) end # Set given options. @@ -231,6 +233,10 @@ def initialize(options = {}) # @see initialize def set_attributes(options) options.each_pair do |key, value| + if key == :file || key == :writedata + # TODO: modes other than 'w+' + value = open_file(value, 'w+') + end method = "#{key}=" unless respond_to?(method) raise Errors::InvalidOption.new(key) @@ -252,6 +258,7 @@ def reset @on_body = nil @procs = nil @mirror = nil + close_all_files Curl.easy_reset(handle) set_callbacks end diff --git a/lib/ethon/easy/callbacks.rb b/lib/ethon/easy/callbacks.rb index 19f630ee..08f8c789 100644 --- a/lib/ethon/easy/callbacks.rb +++ b/lib/ethon/easy/callbacks.rb @@ -18,11 +18,18 @@ def self.included(base) # # @example Set callbacks. # easy.set_callbacks - def set_callbacks - Curl.set_option(:writefunction, body_write_callback, handle) + def set_callbacks(options = {}) + if options[:file].nil? && options[:writedata].nil? + Curl.set_option(:writefunction, body_write_callback, handle) + @response_body = "" + else + # If we are writing to a file, set writefuntion to NULL + Curl.set_option(:writefunction, nil, handle) + @response_body = nil + end Curl.set_option(:headerfunction, header_write_callback, handle) Curl.set_option(:debugfunction, debug_callback, handle) - @response_body = "" + on_complete { close_all_files } @response_headers = "" @headers_called = false @debug_info = Ethon::Easy::DebugInfo.new diff --git a/lib/ethon/easy/files.rb b/lib/ethon/easy/files.rb new file mode 100644 index 00000000..af3b9cff --- /dev/null +++ b/lib/ethon/easy/files.rb @@ -0,0 +1,18 @@ +module Ethon + class Easy + # This module contains logic for managing open files on an easy instance + module Files + def open_file(path, mode) + @open_files ||= [] + fh = Libc.ffi_fopen(path, mode) + @open_files << fh + fh + end + + def close_all_files + @open_files.each {|fh| Libc.ffi_fclose(fh)} unless @open_files.nil? + @open_files = [] + end + end + end +end diff --git a/lib/ethon/libc.rb b/lib/ethon/libc.rb index 84598667..92a93d48 100644 --- a/lib/ethon/libc.rb +++ b/lib/ethon/libc.rb @@ -12,9 +12,22 @@ def self.windows? Gem.win_platform? end + def self.ffi_fclose(fh) + retv = fclose(fh) + fail SystemCallError.new(FFI.errno) unless retv == 0 + end + + def self.ffi_fopen(path, mode) + fh = fopen(path, mode) + fail SystemCallError.new(FFI.errno), path if fh == nil + return fh + end + unless windows? attach_function :getdtablesize, [], :int attach_function :free, [:pointer], :void + attach_function :fopen, [:string, :string], :pointer + attach_function :fclose, [:pointer], :int end end end From dba96b9e76b7514fd7ad6c7481196bbf815028ad Mon Sep 17 00:00:00 2001 From: Seth Madison Date: Mon, 9 Feb 2015 11:13:20 -0500 Subject: [PATCH 2/3] Fix tests: we always close_all_files in on_complete --- spec/ethon/easy/response_callbacks_spec.rb | 8 +++++--- spec/ethon/easy_spec.rb | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/spec/ethon/easy/response_callbacks_spec.rb b/spec/ethon/easy/response_callbacks_spec.rb index 54ae81ae..bcc9f2f8 100644 --- a/spec/ethon/easy/response_callbacks_spec.rb +++ b/spec/ethon/easy/response_callbacks_spec.rb @@ -11,22 +11,24 @@ context "when no block given" do it "returns @#{callback_type}" do - expect(easy.send("#{callback_type}")).to eq([]) + expect(easy.send("#{callback_type}")).to be_kind_of(Array) end end context "when block given" do it "stores" do + orig_size = easy.send(callback_type).count easy.send(callback_type) { p 1 } - expect(easy.instance_variable_get("@#{callback_type}").size).to eq(1) + expect(easy.instance_variable_get("@#{callback_type}").size).to eq(orig_size + 1) end end context "when multiple blocks given" do it "stores" do + orig_size = easy.send(callback_type).count easy.send(callback_type) { p 1 } easy.send(callback_type) { p 2 } - expect(easy.instance_variable_get("@#{callback_type}").size).to eq(2) + expect(easy.instance_variable_get("@#{callback_type}").size).to eq(orig_size + 2) end end end diff --git a/spec/ethon/easy_spec.rb b/spec/ethon/easy_spec.rb index 4017986b..e8729b9c 100644 --- a/spec/ethon/easy_spec.rb +++ b/spec/ethon/easy_spec.rb @@ -75,9 +75,10 @@ end it "resets on_complete" do + orig_size = easy.on_complete.count easy.on_complete { p 1 } easy.reset - expect(easy.on_complete).to be_empty + expect(easy.on_complete.count).to eq(orig_size) end it "resets on_headers" do From fd45295be1dbfdfad7595b9931b1b67cb7cb6711 Mon Sep 17 00:00:00 2001 From: Seth Madison Date: Mon, 9 Feb 2015 13:22:25 -0500 Subject: [PATCH 3/3] Add some tests for new :file and :writedata code --- spec/ethon/easy/callbacks_spec.rb | 7 +++++++ spec/ethon/easy/files_spec.rb | 30 ++++++++++++++++++++++++++++++ spec/ethon/easy_spec.rb | 15 +++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 spec/ethon/easy/files_spec.rb diff --git a/spec/ethon/easy/callbacks_spec.rb b/spec/ethon/easy/callbacks_spec.rb index a21ba118..2bb151c0 100644 --- a/spec/ethon/easy/callbacks_spec.rb +++ b/spec/ethon/easy/callbacks_spec.rb @@ -12,6 +12,13 @@ easy.set_callbacks end + [:file, :writedata].each do |file_opt| + it "sets @response_body to null if file passed" do + easy.set_callbacks({file_opt => '/some/file'}) + expect(easy.instance_variable_get(:@response_body)).to be_nil + end + end + it "resets @response_body" do easy.set_callbacks expect(easy.instance_variable_get(:@response_body)).to eq("") diff --git a/spec/ethon/easy/files_spec.rb b/spec/ethon/easy/files_spec.rb new file mode 100644 index 00000000..2b617f11 --- /dev/null +++ b/spec/ethon/easy/files_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' + +describe Ethon::Easy::Files do + let!(:easy) { Ethon::Easy.new } + + describe "#open_file" do + path, mode = '/i/am/no/file', 'w' + + before do + expect(Ethon::Libc).to receive(:ffi_fopen).with(path, mode) + end + + it "calls ffi_fopen" do + easy.open_file(path, mode) + end + + it "tracks the open files in @open_files" do + easy.open_file(path, mode) + expect(easy.instance_variable_get(:@open_files).count).to eq(1) + end + end + + describe "#close_all_files" do + it "calls ffi_fclose on all open files" do + easy.instance_variable_set(:@open_files, [nil]) + expect(Ethon::Libc).to receive(:ffi_fclose).exactly(1).times + easy.close_all_files + end + end +end diff --git a/spec/ethon/easy_spec.rb b/spec/ethon/easy_spec.rb index e8729b9c..a61979e2 100644 --- a/spec/ethon/easy_spec.rb +++ b/spec/ethon/easy_spec.rb @@ -53,6 +53,16 @@ expect{ easy.set_attributes({:fubar => 1}) }.to raise_error(Ethon::Errors::InvalidOption) end end + + context "when file passed" do + it "opens the file" do + [:file, :writedata].each do |file_opt| + path = '/i/am/not/a/file' + expect(easy).to receive(:open_file).with(path, 'w+') + easy.set_attributes({file_opt => path}) + end + end + end end end @@ -87,6 +97,11 @@ expect(easy.on_headers).to be_empty end + it "closes files" do + easy.reset + expect(easy.instance_variable_get(:@open_files)).to eq([]) + end + it "resets on_body" do easy.on_body { p 1 } easy.reset