diff --git a/courlan/urlutils.py b/courlan/urlutils.py index 3f5692d..b03c492 100644 --- a/courlan/urlutils.py +++ b/courlan/urlutils.py @@ -108,9 +108,13 @@ def fix_relative_urls(baseurl: str, url: str) -> str: if url.startswith("{"): return url - parsed_base = urlsplit(baseurl) + try: + parsed_base = urlsplit(baseurl) + split_url = urlsplit(url) + except ValueError: + return url + base_netloc = parsed_base.netloc - split_url = urlsplit(url) if split_url.netloc not in (base_netloc, ""): if split_url.scheme: diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 4ca5120..661d67c 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -151,6 +151,8 @@ def test_fix_relative(): == "https://www.example.org/foo.html?q=bar#baz" ) assert fix_relative_urls("https://www.example.org", "{privacy}") == "{privacy}" + # malformed IPv6 in protocol-relative href must not raise (urlsplit, not _parse) + assert fix_relative_urls("https://example.org", "//[::1/path") == "//[::1/path" def test_scrub(): @@ -984,6 +986,11 @@ def test_extraction(): extract_links(None, base_url="https://test.com/", external_bool=False) assert not extract_links(None, url="https://test.com/", external_bool=False) assert not extract_links("", "https://test.com/", False) + # malformed IPv6 protocol-relative href must not raise (fix_relative_urls path) + pagecontent = 'xy' + assert extract_links(pagecontent, "https://example.com/", False) == { + "https://example.com/ok" + } # anchor tags matched by the regex but without an href yield no candidate pagecontent = 'y' assert not extract_links(pagecontent, "https://test.com/", False)