Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions courlan/filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from .hosts import _canonical_ip, _idna_encode
from .langcodes import langcodes_score
from .urlutils import _strip_trailing_dot

LOGGER = logging.getLogger(__name__)

Expand Down Expand Up @@ -128,6 +129,8 @@ def _valid_port(tail: str) -> bool:

def domain_filter(domain: str) -> bool:
"Find invalid domain/host names."
# FQDN absolute form ("example.com.") is valid; extract_domain already strips it
domain = _strip_trailing_dot(domain)
# no valid FQDN exceeds the DNS length limit
if len(domain) > 253:
return False
Expand Down
9 changes: 7 additions & 2 deletions courlan/urlutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,13 @@


def _strip_trailing_dot(host: str) -> str:
"Drop a single trailing dot (FQDN form); multiple trailing dots stay invalid."
return host[:-1] if host.endswith(".") and not host.endswith("..") else host
"Drop a single trailing FQDN dot from host or host:port; multiple dots stay invalid."
if host.endswith(".") and not host.endswith(".."):
return host[:-1]
head, sep, tail = host.rpartition(":")
if sep and "@" not in host and head.endswith(".") and not head.endswith(".."):
return f"{head[:-1]}:{tail}"
return host


def get_tldinfo(url: str) -> tuple[str | None, str | None]:
Expand Down
19 changes: 19 additions & 0 deletions tests/unit_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,16 @@ def test_urlcheck_domain():
# bare suffix has no registrable domain, though domain_filter alone accepts it
assert check_url("https://a.ck/page") is None
assert check_url("https://co.uk/page") is None
# trailing-dot FQDN URLs are valid; domain matches extract_domain stripping
assert check_url("http://example.com./page") == (
"http://example.com./page",
"example.com",
)
assert check_url("http://192.168.0.1./x") == (
"http://192.168.0.1./x",
"192.168.0.1",
)
assert check_url("http://example.com../page") is None


def test_urlcheck_port():
Expand Down Expand Up @@ -874,6 +884,15 @@ def test_domain_filter():
assert domain_filter("[::1]:99999") is False
assert domain_filter("[not-an-ip]") is False
assert domain_filter("[::1") is False
# trailing-dot FQDN (absolute DNS form) matches extract_domain, not false-reject
assert domain_filter("example.com.") is True
assert domain_filter("www.example.com.") is True
assert domain_filter("example.com.:8080") is True
assert domain_filter("192.168.0.1.") is True
assert domain_filter("192.168.0.1.:8080") is True
assert (
domain_filter("example.com..") is False
) # multiple trailing dots stay invalid

# hex-only strings that are not IPs must still be validated as domains
assert domain_filter("abc.de") is True
Expand Down
Loading