From 3b7c9e3067238dc416f2da2053e239a148f6d207 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:31:03 -0700 Subject: [PATCH 1/2] fix: skip malformed IPv6 URLs in extract_links helpers urlsplit raises ValueError on unbalanced brackets. Catch that in fix_relative_urls and get_base_url so extract_links can discard bad hrefs the same way check_url already does. --- courlan/urlutils.py | 18 ++++++++++++++---- tests/unit_tests.py | 9 +++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/courlan/urlutils.py b/courlan/urlutils.py index 3f5692d..b06537a 100644 --- a/courlan/urlutils.py +++ b/courlan/urlutils.py @@ -61,7 +61,10 @@ def extract_domain( def _parse(url: str | SplitResult) -> SplitResult: "Parse a string or use urllib.parse object directly." if isinstance(url, str): - parsed_url = urlsplit(unescape(url)) + try: + parsed_url = urlsplit(unescape(url)) + except ValueError as err: + raise ValueError(f"invalid URL: {url}") from err elif isinstance(url, SplitResult): parsed_url = url else: @@ -72,7 +75,10 @@ def _parse(url: str | SplitResult) -> SplitResult: def get_base_url(url: str | SplitResult) -> str: """Strip URL of some of its parts to get base URL. Accepts strings and urllib.parse ParseResult objects.""" - parsed_url = _parse(url) + try: + parsed_url = _parse(url) + except ValueError: + return "" if parsed_url.scheme: scheme = parsed_url.scheme + "://" else: @@ -108,9 +114,13 @@ def fix_relative_urls(baseurl: str, url: str) -> str: if url.startswith("{"): return url - parsed_base = urlsplit(baseurl) + try: + parsed_base = urlsplit(baseurl) + split_url = urlsplit(url) + except ValueError: + return url + base_netloc = parsed_base.netloc - split_url = urlsplit(url) if split_url.netloc not in (base_netloc, ""): if split_url.scheme: diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 4ca5120..6c6807d 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -151,6 +151,9 @@ def test_fix_relative(): == "https://www.example.org/foo.html?q=bar#baz" ) assert fix_relative_urls("https://www.example.org", "{privacy}") == "{privacy}" + # malformed IPv6 in protocol-relative or base URL must not raise + assert fix_relative_urls("https://example.org", "//[::1/path") == "//[::1/path" + assert get_base_url("https://[::1") == "" def test_scrub(): @@ -984,6 +987,12 @@ def test_extraction(): extract_links(None, base_url="https://test.com/", external_bool=False) assert not extract_links(None, url="https://test.com/", external_bool=False) assert not extract_links("", "https://test.com/", False) + # malformed IPv6 protocol-relative href / page URL must not raise + pagecontent = 'xy' + assert extract_links(pagecontent, "https://example.com/", False) == { + "https://example.com/ok" + } + assert extract_links('y', "https://[::1", False) == set() # anchor tags matched by the regex but without an href yield no candidate pagecontent = 'y' assert not extract_links(pagecontent, "https://test.com/", False) From 6973c8051712d6b85c2ce6ce98a4c7d2cd66d85a Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:41:38 -0700 Subject: [PATCH 2/2] fix: narrow to fix_relative_urls only Leave _parse/get_base_url to the existing malformed-URL degrade change. extract_links still crashed on protocol-relative hrefs because those call urlsplit directly in fix_relative_urls. --- courlan/urlutils.py | 10 ++-------- tests/unit_tests.py | 6 ++---- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/courlan/urlutils.py b/courlan/urlutils.py index b06537a..b03c492 100644 --- a/courlan/urlutils.py +++ b/courlan/urlutils.py @@ -61,10 +61,7 @@ def extract_domain( def _parse(url: str | SplitResult) -> SplitResult: "Parse a string or use urllib.parse object directly." if isinstance(url, str): - try: - parsed_url = urlsplit(unescape(url)) - except ValueError as err: - raise ValueError(f"invalid URL: {url}") from err + parsed_url = urlsplit(unescape(url)) elif isinstance(url, SplitResult): parsed_url = url else: @@ -75,10 +72,7 @@ def _parse(url: str | SplitResult) -> SplitResult: def get_base_url(url: str | SplitResult) -> str: """Strip URL of some of its parts to get base URL. Accepts strings and urllib.parse ParseResult objects.""" - try: - parsed_url = _parse(url) - except ValueError: - return "" + parsed_url = _parse(url) if parsed_url.scheme: scheme = parsed_url.scheme + "://" else: diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 6c6807d..661d67c 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -151,9 +151,8 @@ def test_fix_relative(): == "https://www.example.org/foo.html?q=bar#baz" ) assert fix_relative_urls("https://www.example.org", "{privacy}") == "{privacy}" - # malformed IPv6 in protocol-relative or base URL must not raise + # malformed IPv6 in protocol-relative href must not raise (urlsplit, not _parse) assert fix_relative_urls("https://example.org", "//[::1/path") == "//[::1/path" - assert get_base_url("https://[::1") == "" def test_scrub(): @@ -987,12 +986,11 @@ def test_extraction(): extract_links(None, base_url="https://test.com/", external_bool=False) assert not extract_links(None, url="https://test.com/", external_bool=False) assert not extract_links("", "https://test.com/", False) - # malformed IPv6 protocol-relative href / page URL must not raise + # malformed IPv6 protocol-relative href must not raise (fix_relative_urls path) pagecontent = 'xy' assert extract_links(pagecontent, "https://example.com/", False) == { "https://example.com/ok" } - assert extract_links('y', "https://[::1", False) == set() # anchor tags matched by the regex but without an href yield no candidate pagecontent = 'y' assert not extract_links(pagecontent, "https://test.com/", False)