diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index d277a7cbfeb..312871aa5c0 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1434,7 +1434,7 @@ def markdown_extract(text, extract_length=190): will not be truncated.''' if not text: return '' - plain = RE_MD_HTML_TAGS.sub('', markdown(text)) + plain = bleach_clean(markdown(text), strip=True) if not extract_length or len(plain) < extract_length: return literal(plain) @@ -2318,10 +2318,6 @@ def get_request_param(parameter_name, default=None): flags=re.UNICODE ) -# find all tags but ignore < in the strings so that we can use it correctly -# in markdown -RE_MD_HTML_TAGS = re.compile('<[^><]*>') - @core_helper def html_auto_link(data): @@ -2379,7 +2375,6 @@ def render_markdown(data, auto_link=True, allow_html=False): if allow_html: data = markdown(data.strip()) else: - data = RE_MD_HTML_TAGS.sub('', data.strip()) data = bleach_clean( markdown(data), strip=True, tags=MARKDOWN_TAGS, attributes=MARKDOWN_ATTRIBUTES) diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index 89b7da80775..467e7f6f537 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -16,8 +16,9 @@ import ckan.logic as logic from ckan.lib.search.common import ( - SearchIndexError, SearchError, SearchQueryError, - make_connection, is_available, SolrSettings + make_connection, SearchIndexError, SearchQueryError, # type: ignore + SolrConnectionError, # type: ignore + SearchError, is_available, SolrSettings ) from ckan.lib.search.index import PackageSearchIndex, NoopSearchIndex from ckan.lib.search.query import ( diff --git a/ckan/lib/search/common.py b/ckan/lib/search/common.py index f951fcbdca0..c1084d5f26c 100644 --- a/ckan/lib/search/common.py +++ b/ckan/lib/search/common.py @@ -24,6 +24,10 @@ class SearchQueryError(SearchError): pass +class SolrConnectionError(Exception): + pass + + DEFAULT_SOLR_URL = 'http://127.0.0.1:8983/solr' diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 3ffeb4f6ef2..e1fb926714c 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -13,7 +13,7 @@ from ckan.common import config, aslist from ckan.lib.search.common import ( - make_connection, SearchError, SearchQueryError + make_connection, SearchError, SearchQueryError, SolrConnectionError ) log = logging.getLogger(__name__) @@ -427,6 +427,12 @@ def _check_query_parser(param, value): "Can't determine Sort Order" in e.args[0] or \ 'Unknown sort order' in e.args[0]: raise SearchQueryError('Invalid "sort" parameter') + + if ("Failed to connect to server" in e.args[0] or + "Connection to server" in e.args[0]): + log.warning("Connection Error: Failed to connect to Solr server.") + raise SolrConnectionError("Solr returned an error while searching.") + raise SearchError('SOLR returned an error running query: %r Error: %r' % (query, e)) self.count = solr_response.hits diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 8d4846d216c..c73d1682dbd 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -284,3 +284,12 @@ def test_i18n_only_known_locales_are_accepted(app): url = url_for("api.i18n_js_translations", ver=2, lang="unknown_lang") r = app.get(url, status=400) assert "Bad request - Unknown locale" in r.get_data(as_text=True) + + +@pytest.mark.ckan_config("solr_url", "https://xxxx/notofund") +def test_package_search_connection_errors(app): + + res = app.get( + url_for("api.action", logic_function="package_search", ver=3), + ) + assert res.json["error"]["__type"] == "Search Connection Error" diff --git a/ckan/tests/lib/test_helpers.py b/ckan/tests/lib/test_helpers.py index 3613c322d1a..d3bfd8f14dd 100644 --- a/ckan/tests/lib/test_helpers.py +++ b/ckan/tests/lib/test_helpers.py @@ -295,8 +295,8 @@ class TestHelpersRenderMarkdown(object): @pytest.mark.parametrize( "data,output,allow_html", [ - ("
moo
", False), + ("", "", True), + ("", "moo", False), ( "http://example.com", '', @@ -320,7 +320,7 @@ class TestHelpersRenderMarkdown(object): (u"[text](javascript: alert(1))", u"", False), ( u'
and text
and text
", + '
and text
somelink
", + "somelink", + '', False, ), ], diff --git a/ckan/views/api.py b/ckan/views/api.py index dadfbe0e2fb..5ecf7e6889a 100644 --- a/ckan/views/api.py +++ b/ckan/views/api.py @@ -19,7 +19,9 @@ from ckan.lib.navl.dictization_functions import DataError from ckan.logic import get_action, ValidationError, NotFound, NotAuthorized -from ckan.lib.search import SearchError, SearchIndexError, SearchQueryError +from ckan.lib.search import ( + SearchError, SearchIndexError, SearchQueryError, SolrConnectionError +) log = logging.getLogger(__name__) @@ -351,6 +353,12 @@ def action(logic_function, ver=API_DEFAULT_VERSION): str(e)} return_dict[u'success'] = False return _finish(500, return_dict, content_type=u'json') + except SolrConnectionError: + return_dict[u'error'] = { + u'__type': u'Search Connection Error', + u'message': u'Unable to connect to the search server'} + return_dict[u'success'] = False + return _finish(500, return_dict, content_type=u'json') except Exception as e: return_dict[u'error'] = { u'__type': u'Internal Server Error', diff --git a/ckan/views/dataset.py b/ckan/views/dataset.py index 5f32faa221b..000695e46a4 100644 --- a/ckan/views/dataset.py +++ b/ckan/views/dataset.py @@ -24,7 +24,9 @@ from ckan.views.home import CACHE_PARAMETERS from ckan.lib.plugins import lookup_package_plugin from ckan.lib.render import TemplateNotFound -from ckan.lib.search import SearchError, SearchQueryError, SearchIndexError +from ckan.lib.search import ( + SearchError, SearchQueryError, SearchIndexError, SolrConnectionError +) from ckan.views import LazyView @@ -336,7 +338,10 @@ def search(package_type): _(u'Invalid search query: {error_message}') .format(error_message=str(se)) ) - except SearchError as se: + except (SearchError, SolrConnectionError) as se: + if isinstance(se, SolrConnectionError): + base.abort(500, se.args[0]) + # May be bad input from the user, but may also be more serious like # bad code causing a SOLR syntax error, or a problem connecting to # SOLR