From 3c23d1ff84e99560740f56a46a586a37f985407e Mon Sep 17 00:00:00 2001 From: Matt Zumwalt Date: Wed, 18 Nov 2015 13:35:17 +0100 Subject: [PATCH 1/3] Refines and tests collection create/update/delete commands --- lib/solr_wrapper.rb | 2 + lib/solr_wrapper/instance.rb | 81 ++++++++++++++-- .../{ => conf}/_rest_managed.json | 0 .../basic_configs/{ => conf}/currency.xml | 0 .../{ => conf}/lang/stopwords_en.txt | 0 .../basic_configs/{ => conf}/protwords.txt | 0 .../basic_configs/{ => conf}/schema.xml | 0 .../basic_configs/{ => conf}/solrconfig.xml | 0 .../basic_configs/{ => conf}/stopwords.txt | 0 .../basic_configs/{ => conf}/synonyms.txt | 0 spec/lib/solr_wrapper/instance_spec.rb | 92 ++++++++++++++++++- 11 files changed, 165 insertions(+), 10 deletions(-) rename spec/fixtures/basic_configs/{ => conf}/_rest_managed.json (100%) rename spec/fixtures/basic_configs/{ => conf}/currency.xml (100%) rename spec/fixtures/basic_configs/{ => conf}/lang/stopwords_en.txt (100%) rename spec/fixtures/basic_configs/{ => conf}/protwords.txt (100%) rename spec/fixtures/basic_configs/{ => conf}/schema.xml (100%) rename spec/fixtures/basic_configs/{ => conf}/solrconfig.xml (100%) rename spec/fixtures/basic_configs/{ => conf}/stopwords.txt (100%) rename spec/fixtures/basic_configs/{ => conf}/synonyms.txt (100%) diff --git a/lib/solr_wrapper.rb b/lib/solr_wrapper.rb index 9d46a5a..60ed2d6 100644 --- a/lib/solr_wrapper.rb +++ b/lib/solr_wrapper.rb @@ -2,6 +2,8 @@ require 'solr_wrapper/instance' module SolrWrapper + class CollectionNotFoundError < RuntimeError ; end + class ZookeeperNotRunning < RuntimeError ; end def self.default_solr_version '5.3.1' end diff --git a/lib/solr_wrapper/instance.rb b/lib/solr_wrapper/instance.rb index 07ef501..e5b6fa1 100644 --- a/lib/solr_wrapper/instance.rb +++ b/lib/solr_wrapper/instance.rb @@ -45,6 +45,7 @@ def wrap(&_block) ## # Start Solr and wait for it to become available + # @return [StringIO] output from executing the command def start extract_and_configure if managed? @@ -59,6 +60,7 @@ def start ## # Stop Solr and wait for it to finish exiting + # @return [StringIO] output from executing the command def stop if managed? && started? @@ -74,12 +76,23 @@ def stop ## # Stop Solr and wait for it to finish exiting + # @return [StringIO] output from executing the command def restart if managed? && started? exec('restart', p: port, c: options[:cloud]) end end + ## + # Stop solr and remove the install directory + # Warning: This will delete the entire instance_dir + # @return [String] path to the instance_dir that was deleted + def destroy + stop + FileUtils.rm_rf instance_dir + instance_dir + end + ## # Check the status of a managed Solr service def status @@ -91,33 +104,79 @@ def status ## # Is Solr running? + # @return [Boolean] whether solr is running def started? !!status end ## - # Create a new collection in solr + # Create a new collection (or core) in solr + # @param [String] name of the collection to create (defaults to a generated hex value) # @param [Hash] options - # @option options [String] :name # @option options [String] :dir - def create(options = {}) - options[:name] ||= SecureRandom.hex - + # @return [String] name of the collection created + def create(name=nil, options = {}) + name ||= SecureRandom.hex create_options = { p: port } - create_options[:c] = options[:name] if options[:name] + create_options[:c] = name create_options[:d] = options[:dir] if options[:dir] exec("create", create_options) - options[:name] + name end ## - # Create a new collection in solr + # Delete a collection (or core) from solr # @param [String] name collection name + # @return [StringIO] output from executing the command def delete(name, _options = {}) exec("delete", c: name, p: port) end + ## + # Create or Update a collection (or core) in solr + # It is not possible to 'update' a core. You have + # to delete it and create again. + # @param [String] name collection name + # @option options [String] :dir + # @return [String] name of the collection + def create_or_update(name, options={}) + delete(name, options) if collection_exists?(name) + create(name, options) + end + + ### + # Check whether a collection (or core) exists in solr + # @param [String] name collection name + def collection_exists?(name) + begin + # Delete the collection if it exists + healthcheck(name) + true + rescue SolrWrapper::CollectionNotFoundError + false + end + end + + ### + # Run solr healthcheck command for a collection (or core) + # @param [String] name collection name + def healthcheck(name, _options = {}) + begin + exec("healthcheck", c: name, z:"#{host}:#{zkport}") + rescue RuntimeError => e + case e.message + when /ERROR: Collection #{name} not found!/ + raise SolrWrapper::CollectionNotFoundError, e.message + when /Could not connect to ZooKeeper/, /port out of range/, /org.apache.zookeeper.ClientCnxn\$SendThread; Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect/ + raise SolrWrapper::ZookeeperNotRunning, "Zookeeper is not running at #{host}:#{zkport}. Are you sure solr is running in cloud mode?" + else + raise e + end + end + end + + ## # Create a new collection, run the block, and then clean up the collection # @param [Hash] options @@ -126,7 +185,7 @@ def delete(name, _options = {}) def with_collection(options = {}) return yield if options.empty? - name = create(options) + name = create(options[:name], options) begin yield name ensure @@ -146,6 +205,10 @@ def port @port ||= options.fetch(:port, random_open_port).to_s end + def zkport + @zkport ||= (port.to_i + 1000).to_s + end + ## # Clean up any files solr_wrapper may have downloaded def clean! diff --git a/spec/fixtures/basic_configs/_rest_managed.json b/spec/fixtures/basic_configs/conf/_rest_managed.json similarity index 100% rename from spec/fixtures/basic_configs/_rest_managed.json rename to spec/fixtures/basic_configs/conf/_rest_managed.json diff --git a/spec/fixtures/basic_configs/currency.xml b/spec/fixtures/basic_configs/conf/currency.xml similarity index 100% rename from spec/fixtures/basic_configs/currency.xml rename to spec/fixtures/basic_configs/conf/currency.xml diff --git a/spec/fixtures/basic_configs/lang/stopwords_en.txt b/spec/fixtures/basic_configs/conf/lang/stopwords_en.txt similarity index 100% rename from spec/fixtures/basic_configs/lang/stopwords_en.txt rename to spec/fixtures/basic_configs/conf/lang/stopwords_en.txt diff --git a/spec/fixtures/basic_configs/protwords.txt b/spec/fixtures/basic_configs/conf/protwords.txt similarity index 100% rename from spec/fixtures/basic_configs/protwords.txt rename to spec/fixtures/basic_configs/conf/protwords.txt diff --git a/spec/fixtures/basic_configs/schema.xml b/spec/fixtures/basic_configs/conf/schema.xml similarity index 100% rename from spec/fixtures/basic_configs/schema.xml rename to spec/fixtures/basic_configs/conf/schema.xml diff --git a/spec/fixtures/basic_configs/solrconfig.xml b/spec/fixtures/basic_configs/conf/solrconfig.xml similarity index 100% rename from spec/fixtures/basic_configs/solrconfig.xml rename to spec/fixtures/basic_configs/conf/solrconfig.xml diff --git a/spec/fixtures/basic_configs/stopwords.txt b/spec/fixtures/basic_configs/conf/stopwords.txt similarity index 100% rename from spec/fixtures/basic_configs/stopwords.txt rename to spec/fixtures/basic_configs/conf/stopwords.txt diff --git a/spec/fixtures/basic_configs/synonyms.txt b/spec/fixtures/basic_configs/conf/synonyms.txt similarity index 100% rename from spec/fixtures/basic_configs/synonyms.txt rename to spec/fixtures/basic_configs/conf/synonyms.txt diff --git a/spec/lib/solr_wrapper/instance_spec.rb b/spec/lib/solr_wrapper/instance_spec.rb index 5300bdc..884db5a 100644 --- a/spec/lib/solr_wrapper/instance_spec.rb +++ b/spec/lib/solr_wrapper/instance_spec.rb @@ -17,6 +17,97 @@ end end end + describe 'destroy' do + subject { solr_instance.destroy } + it 'stops solr and deletes the entire instance_dir' do + expect(solr_instance).to receive(:stop) + expect(FileUtils).to receive(:rm_rf).with(solr_instance.instance_dir) + subject + end + end + describe 'cloud commands' do + let(:collection_name) { 'test_collection' } + let(:existing_collection) { solr_instance.create(collection_name) } + let(:collection_config_dir) { File.join(FIXTURES_DIR, "basic_configs") } + let(:solr_instance) { @solr_instance } + before(:all) do + @solr_instance = SolrWrapper::Instance.new(cloud: true) + @solr_instance.start + end + after(:all) do + @solr_instance.stop + end + describe 'create' do + subject { solr_instance.create(collection_name, dir:collection_config_dir) } + after { solr_instance.delete(collection_name) } + it 'creates a collection' do + expect(solr_instance.collection_exists?(collection_name)).to eq false + expect(subject).to eq collection_name + expect(solr_instance.collection_exists?(collection_name)).to eq true + end + end + describe 'delete' do + subject { solr_instance.delete(existing_collection) } + it 'deletes a collection' do + expect(solr_instance.collection_exists?(existing_collection)).to eq true + subject + expect(solr_instance.collection_exists?(existing_collection)).to eq false + end + end + describe 'create_or_update' do + subject { solr_instance.create_or_update(collection_name, dir:collection_config_dir) } + context 'when the collection does not exist' do + before do + expect(solr_instance).to receive(:collection_exists?).and_return(false) + end + it 'creates the collection' do + expect(solr_instance).to_not receive(:delete) + expect(solr_instance).to receive(:create).with(collection_name, dir:collection_config_dir) + subject + end + end + context 'when the collection already exists' do + before do + expect(solr_instance).to receive(:collection_exists?).and_return(true) + end + it 'delete the collection and then creates it again' do + expect(solr_instance).to receive(:delete).with(collection_name, dir:collection_config_dir) + expect(solr_instance).to receive(:create).with(collection_name, dir:collection_config_dir) + subject + end + end + end + describe 'healthcheck' do + context 'when the collection does not exist' do + subject { solr_instance.healthcheck('nonexistent') } + it 'raises an error' do + expect { subject }.to raise_error(SolrWrapper::CollectionNotFoundError) + end + end + context 'when the collection exists' do + subject { solr_instance.healthcheck(existing_collection) } + after { solr_instance.delete(existing_collection) } + it 'returns info about the collection' do + expect(subject).to be_instance_of StringIO + json = JSON.parse(subject.read) + expect(json['collection']).to eq existing_collection + expect(json['status']).to eq 'healthy' + expect(json['numDocs']).to eq 0 + end + end + context 'when zookeeper is not running' do + let(:wrapper_error_message) { "Zookeeper is not running at #{solr_instance.host}:#{solr_instance.zkport}. Are you sure solr is running in cloud mode?" } + it 'raises an appropriate error' do + expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "ERROR: java.lang.IllegalArgumentException: port out of range:65831") + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "org.apache.zookeeper.ClientCnxn$SendThread; Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect") + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + expect(solr_instance).to receive(:exec).and_raise(RuntimeError, "ERROR: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:58499 within 10000 ms") + expect{ solr_instance.healthcheck('foo') }.to raise_error(SolrWrapper::ZookeeperNotRunning, wrapper_error_message) + end + end + end + end describe 'exec' do let(:cmd) { 'start' } let(:options) { { p: '4098', help: true } } @@ -29,7 +120,6 @@ result_io = solr_instance.send(:exec, 'start', p: '4098', help: true) expect(result_io.read).to include('Usage: solr start') end - describe 'when something goes wrong' do let(:cmd) { 'healthcheck' } let(:options) { { z: 'localhost:5098' } } From 8a475b1768ee545cad26d57f4f3c4be904a5f594 Mon Sep 17 00:00:00 2001 From: Matt Zumwalt Date: Wed, 25 Nov 2015 12:28:14 +0100 Subject: [PATCH 2/3] Makes configure task run more smoothly --- .gitignore | 1 + lib/solr_wrapper/instance.rb | 8 +++++++- lib/solr_wrapper/tasks/solr_wrapper.rake | 1 + spec/lib/solr_wrapper/instance_spec.rb | 24 ++++++++++++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4bbf540..15031c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ Gemfile.lock *.gem *.rbc +.byebug_history /.config /coverage/ /InstalledFiles diff --git a/lib/solr_wrapper/instance.rb b/lib/solr_wrapper/instance.rb index e5b6fa1..c5456cc 100644 --- a/lib/solr_wrapper/instance.rb +++ b/lib/solr_wrapper/instance.rb @@ -235,7 +235,13 @@ def url def configure raise_error_unless_extracted FileUtils.cp options[:solr_xml], File.join(instance_dir, 'server', 'solr', 'solr.xml') if options[:solr_xml] - FileUtils.cp_r File.join(options[:extra_lib_dir], '.'), File.join(instance_dir, 'server', 'solr', 'lib') if options[:extra_lib_dir] + if options[:extra_lib_dir] + if File.exist?(options[:extra_lib_dir]) + FileUtils.cp_r File.join(options[:extra_lib_dir], '.'), File.join(instance_dir, 'server', 'solr', 'lib') + else + puts "You specified #{options[:extra_lib_dir]} as the :extra_lib_dir but that directory does not exist!" + end + end end def instance_dir diff --git a/lib/solr_wrapper/tasks/solr_wrapper.rake b/lib/solr_wrapper/tasks/solr_wrapper.rake index 2e08ac0..67b6725 100644 --- a/lib/solr_wrapper/tasks/solr_wrapper.rake +++ b/lib/solr_wrapper/tasks/solr_wrapper.rake @@ -11,6 +11,7 @@ namespace :solr do desc 'Install a clean version of solr. Replaces the existing copy if there is one.' task clean: :environment do puts "Installing clean version of solr at #{File.expand_path(@solr_instance.instance_dir)}" + @solr_instance.stop @solr_instance.remove_instance_dir! @solr_instance.extract_and_configure end diff --git a/spec/lib/solr_wrapper/instance_spec.rb b/spec/lib/solr_wrapper/instance_spec.rb index 884db5a..c4b5024 100644 --- a/spec/lib/solr_wrapper/instance_spec.rb +++ b/spec/lib/solr_wrapper/instance_spec.rb @@ -17,6 +17,30 @@ end end end + describe 'configure' do + subject { solr_instance.configure } + context 'when extra_lib_dir is specified' do + before { solr_instance.options[:extra_lib_dir] = 'foo/bar' } + it 'copies the contents of extra_lib_dir into the solr/lib directory' do + expect(File).to receive(:exist?).with('foo/bar').and_return(true) + expect(FileUtils).to receive(:cp_r).with('foo/bar/.', "#{solr_instance.instance_dir}/server/solr/lib") + subject + end + it 'does not try to copy anything if extra_lib_dir does not exist' do + expect(File).to receive(:exist?).with('foo/bar').and_return(false) + expect(FileUtils).to_not receive(:cp_r) + allow(solr_instance).to receive(:puts) + subject + end + end + context 'when extra_lib_dir is not specified' do + it 'does not try to copy anything' do + expect(File).to_not receive(:exist?) + expect(FileUtils).to_not receive(:cp_r) + subject + end + end + end describe 'destroy' do subject { solr_instance.destroy } it 'stops solr and deletes the entire instance_dir' do From d8de2b20aeb5761f7e4a75e373df96081f8e2c03 Mon Sep 17 00:00:00 2001 From: Matt Zumwalt Date: Thu, 26 Nov 2015 09:56:28 +0100 Subject: [PATCH 3/3] (minor) test coverage for SolrWrapper.default_instance --- spec/lib/solr_wrapper_spec.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/spec/lib/solr_wrapper_spec.rb b/spec/lib/solr_wrapper_spec.rb index e420984..96491f6 100644 --- a/spec/lib/solr_wrapper_spec.rb +++ b/spec/lib/solr_wrapper_spec.rb @@ -13,6 +13,29 @@ end end + describe '.default_instance' do + let(:custom_options) { + { + verbose: true, + cloud: true, + port: '8983', + version: '5.3.1', + instance_dir: 'solr', + extra_lib_dir: File.join('myconfigs','lib'), + } + } + subject { SolrWrapper.default_instance } + context 'when @default_instance is not set' do + before do + SolrWrapper.remove_instance_variable(:@default_instance) + end + it "uses default_instance_options" do + SolrWrapper.default_instance_options = custom_options + expect(subject.options).to eq custom_options + end + end + end + describe ".default_instance_options=" do it "sets default options" do SolrWrapper.default_instance_options = { port: '1234' }