From 85903680af77485d581b33e94819c67a2880e3c3 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:29:18 -0700 Subject: [PATCH 1/2] Accept trailing-dot FQDN hosts in domain_filter extract_domain already strips a single trailing DNS dot, but check_url rejected the same hosts via domain_filter, dropping valid crawl URLs. --- courlan/filters.py | 12 ++++++++++++ tests/unit_tests.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/courlan/filters.py b/courlan/filters.py index 96b8a16..6ccfc99 100644 --- a/courlan/filters.py +++ b/courlan/filters.py @@ -121,8 +121,20 @@ def basic_filter(url: str) -> bool: return bool(url.startswith("http") and 10 <= len(url) < 500) +def _strip_fqdn_dot(domain: str) -> str: + "Strip a single trailing FQDN dot from host or host:port (same as extract_domain)." + if domain.endswith(".") and not domain.endswith(".."): + return domain[:-1] + head, sep, tail = domain.rpartition(":") + if sep and "@" not in domain and head.endswith(".") and not head.endswith(".."): + return f"{head[:-1]}:{tail}" + return domain + + def domain_filter(domain: str) -> bool: "Find invalid domain/host names." + # FQDN absolute form ("example.com.") is valid; extract_domain already strips it + domain = _strip_fqdn_dot(domain) # no valid FQDN exceeds the DNS length limit if len(domain) > 253: return False diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 2983d5b..4d5f623 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -796,6 +796,16 @@ def test_urlcheck_domain(): # bare suffix has no registrable domain, though domain_filter alone accepts it assert check_url("https://a.ck/page") is None assert check_url("https://co.uk/page") is None + # trailing-dot FQDN URLs are valid; domain matches extract_domain stripping + assert check_url("http://example.com./page") == ( + "http://example.com./page", + "example.com", + ) + assert check_url("http://192.168.0.1./x") == ( + "http://192.168.0.1./x", + "192.168.0.1", + ) + assert check_url("http://example.com../page") is None def test_urlcheck_port(): @@ -851,6 +861,15 @@ def test_domain_filter(): assert domain_filter("1.2.3.4:99999") is False assert domain_filter("1.2.3.4:0") is False assert domain_filter("1.2.3.4:0080") is False # leading zero rejected + # trailing-dot FQDN (absolute DNS form) matches extract_domain, not false-reject + assert domain_filter("example.com.") is True + assert domain_filter("www.example.com.") is True + assert domain_filter("example.com.:8080") is True + assert domain_filter("192.168.0.1.") is True + assert domain_filter("192.168.0.1.:8080") is True + assert ( + domain_filter("example.com..") is False + ) # multiple trailing dots stay invalid # hex-only strings that are not IPs must still be validated as domains assert domain_filter("abc.de") is True From c9050206cf144bf38f5263dff29bcd72df8daba8 Mon Sep 17 00:00:00 2001 From: santhreal <64453045+santhreal@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:30:28 -0700 Subject: [PATCH 2/2] Accept trailing-dot FQDN hosts in domain_filter extract_domain already strips a single trailing DNS dot, but check_url rejected the same hosts via domain_filter, dropping valid crawl URLs. Reuse and extend _strip_trailing_dot for host and host:port forms. --- courlan/filters.py | 13 ++----------- courlan/urlutils.py | 9 +++++++-- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/courlan/filters.py b/courlan/filters.py index 6ccfc99..9753621 100644 --- a/courlan/filters.py +++ b/courlan/filters.py @@ -8,6 +8,7 @@ from .hosts import _canonical_ip, _idna_encode from .langcodes import langcodes_score +from .urlutils import _strip_trailing_dot LOGGER = logging.getLogger(__name__) @@ -121,20 +122,10 @@ def basic_filter(url: str) -> bool: return bool(url.startswith("http") and 10 <= len(url) < 500) -def _strip_fqdn_dot(domain: str) -> str: - "Strip a single trailing FQDN dot from host or host:port (same as extract_domain)." - if domain.endswith(".") and not domain.endswith(".."): - return domain[:-1] - head, sep, tail = domain.rpartition(":") - if sep and "@" not in domain and head.endswith(".") and not head.endswith(".."): - return f"{head[:-1]}:{tail}" - return domain - - def domain_filter(domain: str) -> bool: "Find invalid domain/host names." # FQDN absolute form ("example.com.") is valid; extract_domain already strips it - domain = _strip_fqdn_dot(domain) + domain = _strip_trailing_dot(domain) # no valid FQDN exceeds the DNS length limit if len(domain) > 253: return False diff --git a/courlan/urlutils.py b/courlan/urlutils.py index 1d6f70e..3601c3d 100644 --- a/courlan/urlutils.py +++ b/courlan/urlutils.py @@ -12,8 +12,13 @@ def _strip_trailing_dot(host: str) -> str: - "Drop a single trailing dot (FQDN form); multiple trailing dots stay invalid." - return host[:-1] if host.endswith(".") and not host.endswith("..") else host + "Drop a single trailing FQDN dot from host or host:port; multiple dots stay invalid." + if host.endswith(".") and not host.endswith(".."): + return host[:-1] + head, sep, tail = host.rpartition(":") + if sep and "@" not in host and head.endswith(".") and not head.endswith(".."): + return f"{head[:-1]}:{tail}" + return host def get_tldinfo(url: str) -> tuple[str | None, str | None]: