Improve LDPath performance for batch transforms

Gemfile

@@ -6,3 +6,5 @@ group :development, :test do
   gem 'coveralls', require: false
   gem 'simplecov', require: false
 end
+
+gem 'lru_redux'

bin/ldpath

@@ -3,6 +3,7 @@
 require 'optparse'
 require 'open-uri'
 require 'ldpath'
+require 'lru_redux'
 
 begin
   require 'rest-client'
@@ -13,11 +14,9 @@ options = {}
 opt_parser = OptionParser.new do |opts|
   opts.banner = "Usage: ldpath [options] URI"
 
-  opts.on("--program=STRING_URI_OR_FILE", "LDPath program to run (reads from STDIN by default)") do |program|
+  opts.on("--program=STRING_URI_OR_FILE", "LDPath program to run") do |program|
     options[:program] = if File.exist?(program) || program =~ /^http/
                           open(program).read
-                        elsif program.strip == "-"
-                          $stdin.read
                         else
                           program
                         end
@@ -26,13 +25,13 @@ end
 
 opt_parser.parse!
 
-uri = ARGV.shift
+uris = ARGV
+uris = ARGF if uris.empty? || uris.first == '-'
 
-if uri.nil?
-  $stderr.puts opt_parser
-  raise OptionParser::MissingArgument, "URI" unless uri
-end
 
-options[:program] ||= $stdin.read
+cache = LruRedux::TTL::Cache.new(1000, 120)
+program = Ldpath::Program.parse(options[:program])
 
-puts Ldpath::Program.parse(options[:program]).evaluate(RDF::URI.new(uri)).to_json
+uris.each do |uri|
+  puts program.evaluate(RDF::URI.new(uri.strip), cache: cache).to_json
+end

lib/ldpath/loaders/linked_data_fragment.rb

@@ -8,18 +8,26 @@ def initialize(endpoint)
   end
 
   def load(uri)
-    Ldpath.logger.debug "Loading LDF data for #{uri.inspect}"
+    i = 0
+    begin
+      Ldpath.logger.debug "Loading LDF data for #{uri.inspect}"
 
-    graph = RDF::Graph.new
-    request_uri = RDF::URI("#{@endpoint}?subject=#{CGI::escape(uri)}")
+      graph = RDF::Graph.new
+      request_uri = RDF::URI("#{@endpoint}?subject=#{CGI::escape(uri)}")
 
-    while request_uri
-      Ldpath.logger.debug " -- querying #{request_uri}"
-      request_graph = RDF::Graph.load(request_uri)
-      graph.insert_statements(request_graph)
-      request_uri = request_graph.first_object([request_uri, NEXT_PAGE, nil])
-    end
+      while request_uri
+        Ldpath.logger.debug " -- querying #{request_uri}"
+        request_graph = RDF::Graph.load(request_uri)
+        graph.insert_statements(request_graph)
+        request_uri = request_graph.first_object([request_uri, NEXT_PAGE, nil])
+      end
 
-    graph
+      graph
+    rescue => e
+      i += 1
+      retry if i < 3
+      Ldpath.logger.warn e
+      RDF::Graph.new
+    end
   end
 end

lib/ldpath/program.rb

@@ -36,8 +36,8 @@ def initialize(mappings, default_loader: Ldpath::Loaders::Direct.new, prefixes:
 
     end
 
-    def evaluate(uri, context: nil, limit_to_context: false)
-      result = Ldpath::Result.new(self, uri, context: context, limit_to_context: limit_to_context)
+    def evaluate(uri, context: nil, limit_to_context: false, cache: RDF::Util::Cache.new)
+      result = Ldpath::Result.new(self, uri, context: context, limit_to_context: limit_to_context, cache: cache)
       unless filters.empty?
         return {} unless filters.all? { |f| f.evaluate(result, uri, result.context) }
       end

spec/ldpath_program_spec.rb

@@ -239,6 +239,19 @@
         result = subject.evaluate RDF::URI.new("http://www.bbc.co.uk/programmes/b0081dq5")
         expect(result["title"]).to match_array "Huw Stephens"
       end
+
+      context 'error handling' do
+        before do
+          stub_request(:get, 'http://example.com/ldf?subject=http://www.bbc.co.uk/programmes/b0081dq5')
+              .to_return(status: 500, body: nil)
+        end
+
+        it "logs an error but continues processing" do
+          expect(Ldpath.logger).to receive(:warn)
+          result = subject.evaluate RDF::URI.new("http://www.bbc.co.uk/programmes/b0081dq5")
+          expect(result["title"]).to match_array []
+        end
+      end
     end
   end