diff --git a/courlan/filters.py b/courlan/filters.py index a563736..0ae4d6a 100644 --- a/courlan/filters.py +++ b/courlan/filters.py @@ -8,6 +8,7 @@ from .hosts import _canonical_ip, _idna_encode from .langcodes import langcodes_score +from .urlutils import _strip_trailing_dot LOGGER = logging.getLogger(__name__) @@ -128,6 +129,8 @@ def _valid_port(tail: str) -> bool: def domain_filter(domain: str) -> bool: "Find invalid domain/host names." + # FQDN absolute form ("example.com.") is valid; extract_domain already strips it + domain = _strip_trailing_dot(domain) # no valid FQDN exceeds the DNS length limit if len(domain) > 253: return False diff --git a/courlan/urlutils.py b/courlan/urlutils.py index 1d6f70e..3601c3d 100644 --- a/courlan/urlutils.py +++ b/courlan/urlutils.py @@ -12,8 +12,13 @@ def _strip_trailing_dot(host: str) -> str: - "Drop a single trailing dot (FQDN form); multiple trailing dots stay invalid." - return host[:-1] if host.endswith(".") and not host.endswith("..") else host + "Drop a single trailing FQDN dot from host or host:port; multiple dots stay invalid." + if host.endswith(".") and not host.endswith(".."): + return host[:-1] + head, sep, tail = host.rpartition(":") + if sep and "@" not in host and head.endswith(".") and not head.endswith(".."): + return f"{head[:-1]}:{tail}" + return host def get_tldinfo(url: str) -> tuple[str | None, str | None]: diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 4376691..19ccc02 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -807,6 +807,16 @@ def test_urlcheck_domain(): # bare suffix has no registrable domain, though domain_filter alone accepts it assert check_url("https://a.ck/page") is None assert check_url("https://co.uk/page") is None + # trailing-dot FQDN URLs are valid; domain matches extract_domain stripping + assert check_url("http://example.com./page") == ( + "http://example.com./page", + "example.com", + ) + assert check_url("http://192.168.0.1./x") == ( + "http://192.168.0.1./x", + "192.168.0.1", + ) + assert check_url("http://example.com../page") is None def test_urlcheck_port(): @@ -874,6 +884,15 @@ def test_domain_filter(): assert domain_filter("[::1]:99999") is False assert domain_filter("[not-an-ip]") is False assert domain_filter("[::1") is False + # trailing-dot FQDN (absolute DNS form) matches extract_domain, not false-reject + assert domain_filter("example.com.") is True + assert domain_filter("www.example.com.") is True + assert domain_filter("example.com.:8080") is True + assert domain_filter("192.168.0.1.") is True + assert domain_filter("192.168.0.1.:8080") is True + assert ( + domain_filter("example.com..") is False + ) # multiple trailing dots stay invalid # hex-only strings that are not IPs must still be validated as domains assert domain_filter("abc.de") is True