Improve Solr connection error handling and improve HTML sanitization

ckan/lib/helpers.py

@@ -1434,7 +1434,7 @@ def markdown_extract(text, extract_length=190):
     will not be truncated.'''
     if not text:
         return ''
-    plain = RE_MD_HTML_TAGS.sub('', markdown(text))
+    plain = bleach_clean(markdown(text), strip=True)
     if not extract_length or len(plain) < extract_length:
         return literal(plain)
 
@@ -2318,10 +2318,6 @@ def get_request_param(parameter_name, default=None):
     flags=re.UNICODE
 )
 
-# find all tags but ignore < in the strings so that we can use it correctly
-# in markdown
-RE_MD_HTML_TAGS = re.compile('<[^><]*>')
-
 
 @core_helper
 def html_auto_link(data):
@@ -2379,7 +2375,6 @@ def render_markdown(data, auto_link=True, allow_html=False):
     if allow_html:
         data = markdown(data.strip())
     else:
-        data = RE_MD_HTML_TAGS.sub('', data.strip())
         data = bleach_clean(
             markdown(data), strip=True,
             tags=MARKDOWN_TAGS, attributes=MARKDOWN_ATTRIBUTES)

ckan/lib/search/__init__.py

@@ -16,8 +16,9 @@
 import ckan.logic as logic
 
 from ckan.lib.search.common import (
-    SearchIndexError, SearchError, SearchQueryError,
-    make_connection, is_available, SolrSettings
+    make_connection, SearchIndexError, SearchQueryError,  # type: ignore
+    SolrConnectionError, # type: ignore
+    SearchError, is_available, SolrSettings
 )
 from ckan.lib.search.index import PackageSearchIndex, NoopSearchIndex
 from ckan.lib.search.query import (

ckan/lib/search/common.py

@@ -24,6 +24,10 @@ class SearchQueryError(SearchError):
     pass
 
 
+class SolrConnectionError(Exception):
+    pass
+
+
 DEFAULT_SOLR_URL = 'http://127.0.0.1:8983/solr'
 
 

ckan/lib/search/query.py

@@ -13,7 +13,7 @@
 
 from ckan.common import config, aslist
 from ckan.lib.search.common import (
-    make_connection, SearchError, SearchQueryError
+    make_connection, SearchError, SearchQueryError, SolrConnectionError
 )
 
 log = logging.getLogger(__name__)
@@ -427,6 +427,12 @@ def _check_query_parser(param, value):
                         "Can't determine Sort Order" in e.args[0] or \
                         'Unknown sort order' in e.args[0]:
                     raise SearchQueryError('Invalid "sort" parameter')
+
+                if ("Failed to connect to server" in e.args[0] or 
+                        "Connection to server" in e.args[0]):
+                    log.warning("Connection Error: Failed to connect to Solr server.")
+                    raise SolrConnectionError("Solr returned an error while searching.")
+
             raise SearchError('SOLR returned an error running query: %r Error: %r' %
                               (query, e))
         self.count = solr_response.hits

ckan/tests/controllers/test_api.py

@@ -284,3 +284,12 @@ def test_i18n_only_known_locales_are_accepted(app):
     url = url_for("api.i18n_js_translations", ver=2, lang="unknown_lang")
     r = app.get(url, status=400)
     assert "Bad request - Unknown locale" in r.get_data(as_text=True)
+
+
+@pytest.mark.ckan_config("solr_url", "https://xxxx/notofund")
+def test_package_search_connection_errors(app):
+
+    res = app.get(
+        url_for("api.action", logic_function="package_search", ver=3),
+    )
+    assert res.json["error"]["__type"] == "Search Connection Error"

ckan/tests/lib/test_helpers.py

@@ -295,8 +295,8 @@ class TestHelpersRenderMarkdown(object):
     @pytest.mark.parametrize(
         "data,output,allow_html",
         [
-            ("<h1>moo</h1>", "<h1>moo</h1>", True),
-            ("<h1>moo</h1>", "<p>moo</p>", False),
+            ("<script>moo</script>", "<script>moo</script>", True),
+            ("<script>moo</script>", "moo", False),
             (
                 "http://example.com",
                 '<p><a href="http://example.com" target="_blank" rel="nofollow">http://example.com</a></p>',
@@ -320,7 +320,7 @@ class TestHelpersRenderMarkdown(object):
             (u"[text](javascript: alert(1))", u"<p><a>text</a></p>", False),
             (
                 u'<p onclick="some.script"><img onmouseover="some.script" src="image.png" /> and text</p>',
-                u"<p>and text</p>",
+                '<p><img src="image.png"> and text</p>',
                 False,
             ),
             (u"#heading", u"<h1>heading</h1>", False),
@@ -416,8 +416,8 @@ class TestHelpersRenderMarkdown(object):
                 False,
             ),
             (
-                u"<a href=\u201dsomelink\u201d>somelink</a>",
-                "<p>somelink</p>",
+                "<a href=\u201dsomelink\u201d>somelink</a>",
+                '<p><a href="\u201dsomelink\u201d">somelink</a></p>',
                 False,
             ),
         ],

ckan/views/api.py

@@ -19,7 +19,9 @@
 
 from ckan.lib.navl.dictization_functions import DataError
 from ckan.logic import get_action, ValidationError, NotFound, NotAuthorized
-from ckan.lib.search import SearchError, SearchIndexError, SearchQueryError
+from ckan.lib.search import (
+    SearchError, SearchIndexError, SearchQueryError, SolrConnectionError
+)
 
 
 log = logging.getLogger(__name__)
@@ -351,6 +353,12 @@ def action(logic_function, ver=API_DEFAULT_VERSION):
                        str(e)}
         return_dict[u'success'] = False
         return _finish(500, return_dict, content_type=u'json')
+    except SolrConnectionError:
+        return_dict[u'error'] = {
+            u'__type': u'Search Connection Error',
+            u'message': u'Unable to connect to the search server'}
+        return_dict[u'success'] = False
+        return _finish(500, return_dict, content_type=u'json')
     except Exception as e:
         return_dict[u'error'] = {
             u'__type': u'Internal Server Error',

ckan/views/dataset.py

@@ -24,7 +24,9 @@
 from ckan.views.home import CACHE_PARAMETERS
 from ckan.lib.plugins import lookup_package_plugin
 from ckan.lib.render import TemplateNotFound
-from ckan.lib.search import SearchError, SearchQueryError, SearchIndexError
+from ckan.lib.search import (
+    SearchError, SearchQueryError, SearchIndexError, SolrConnectionError
+)
 from ckan.views import LazyView
 
 
@@ -336,7 +338,10 @@ def search(package_type):
             _(u'Invalid search query: {error_message}')
             .format(error_message=str(se))
         )
-    except SearchError as se:
+    except (SearchError, SolrConnectionError) as se:
+        if isinstance(se, SolrConnectionError):
+            base.abort(500, se.args[0])
+
         # May be bad input from the user, but may also be more serious like
         # bad code causing a SOLR syntax error, or a problem connecting to
         # SOLR