diff --git a/Gemfile b/Gemfile index 1c4484f..50c1d8f 100644 --- a/Gemfile +++ b/Gemfile @@ -6,3 +6,5 @@ group :development, :test do gem 'coveralls', require: false gem 'simplecov', require: false end + +gem 'lru_redux' diff --git a/bin/ldpath b/bin/ldpath index 8ddf319..35c44e8 100755 --- a/bin/ldpath +++ b/bin/ldpath @@ -3,6 +3,7 @@ require 'optparse' require 'open-uri' require 'ldpath' +require 'lru_redux' begin require 'rest-client' @@ -13,11 +14,9 @@ options = {} opt_parser = OptionParser.new do |opts| opts.banner = "Usage: ldpath [options] URI" - opts.on("--program=STRING_URI_OR_FILE", "LDPath program to run (reads from STDIN by default)") do |program| + opts.on("--program=STRING_URI_OR_FILE", "LDPath program to run") do |program| options[:program] = if File.exist?(program) || program =~ /^http/ open(program).read - elsif program.strip == "-" - $stdin.read else program end @@ -26,13 +25,13 @@ end opt_parser.parse! -uri = ARGV.shift +uris = ARGV +uris = ARGF if uris.empty? || uris.first == '-' -if uri.nil? - $stderr.puts opt_parser - raise OptionParser::MissingArgument, "URI" unless uri -end -options[:program] ||= $stdin.read +cache = LruRedux::TTL::Cache.new(1000, 120) +program = Ldpath::Program.parse(options[:program]) -puts Ldpath::Program.parse(options[:program]).evaluate(RDF::URI.new(uri)).to_json +uris.each do |uri| + puts program.evaluate(RDF::URI.new(uri.strip), cache: cache).to_json +end diff --git a/lib/ldpath/loaders/linked_data_fragment.rb b/lib/ldpath/loaders/linked_data_fragment.rb index e1e357e..d47cb8f 100644 --- a/lib/ldpath/loaders/linked_data_fragment.rb +++ b/lib/ldpath/loaders/linked_data_fragment.rb @@ -8,18 +8,26 @@ def initialize(endpoint) end def load(uri) - Ldpath.logger.debug "Loading LDF data for #{uri.inspect}" + i = 0 + begin + Ldpath.logger.debug "Loading LDF data for #{uri.inspect}" - graph = RDF::Graph.new - request_uri = RDF::URI("#{@endpoint}?subject=#{CGI::escape(uri)}") + graph = RDF::Graph.new + request_uri = RDF::URI("#{@endpoint}?subject=#{CGI::escape(uri)}") - while request_uri - Ldpath.logger.debug " -- querying #{request_uri}" - request_graph = RDF::Graph.load(request_uri) - graph.insert_statements(request_graph) - request_uri = request_graph.first_object([request_uri, NEXT_PAGE, nil]) - end + while request_uri + Ldpath.logger.debug " -- querying #{request_uri}" + request_graph = RDF::Graph.load(request_uri) + graph.insert_statements(request_graph) + request_uri = request_graph.first_object([request_uri, NEXT_PAGE, nil]) + end - graph + graph + rescue => e + i += 1 + retry if i < 3 + Ldpath.logger.warn e + RDF::Graph.new + end end end diff --git a/lib/ldpath/program.rb b/lib/ldpath/program.rb index 99a3870..6b5f65b 100644 --- a/lib/ldpath/program.rb +++ b/lib/ldpath/program.rb @@ -36,8 +36,8 @@ def initialize(mappings, default_loader: Ldpath::Loaders::Direct.new, prefixes: end - def evaluate(uri, context: nil, limit_to_context: false) - result = Ldpath::Result.new(self, uri, context: context, limit_to_context: limit_to_context) + def evaluate(uri, context: nil, limit_to_context: false, cache: RDF::Util::Cache.new) + result = Ldpath::Result.new(self, uri, context: context, limit_to_context: limit_to_context, cache: cache) unless filters.empty? return {} unless filters.all? { |f| f.evaluate(result, uri, result.context) } end diff --git a/spec/ldpath_program_spec.rb b/spec/ldpath_program_spec.rb index 1e059c0..8a4cb0f 100644 --- a/spec/ldpath_program_spec.rb +++ b/spec/ldpath_program_spec.rb @@ -239,6 +239,19 @@ result = subject.evaluate RDF::URI.new("http://www.bbc.co.uk/programmes/b0081dq5") expect(result["title"]).to match_array "Huw Stephens" end + + context 'error handling' do + before do + stub_request(:get, 'http://example.com/ldf?subject=http://www.bbc.co.uk/programmes/b0081dq5') + .to_return(status: 500, body: nil) + end + + it "logs an error but continues processing" do + expect(Ldpath.logger).to receive(:warn) + result = subject.evaluate RDF::URI.new("http://www.bbc.co.uk/programmes/b0081dq5") + expect(result["title"]).to match_array [] + end + end end end