Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 5 additions & 7 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import sys
import threading
import time
from functools import lru_cache
from typing import Any, Callable, Dict, List, Optional, Sequence, Set
from urllib.parse import urlparse

Expand Down Expand Up @@ -321,12 +320,12 @@ def _api_client() -> httpx.Client:
_cache_lock = threading.RLock()


@lru_cache(maxsize=128)
# SECURITY: Do not cache validation results. Caching introduces a TOCTOU (Time-of-Check Time-of-Use)
# vulnerability where a DNS record could change from a public IP to a private IP (DNS Rebinding)
# in the time between validation and fetch. We must re-validate immediately before fetching.
def validate_folder_url(url: str) -> bool:
"""
Validates a folder URL.
Cached to avoid repeated DNS lookups (socket.getaddrinfo) for the same URL
during warm-up and sync phases.
"""
if not url.startswith("https://"):
log.warning(
Expand Down Expand Up @@ -1043,9 +1042,8 @@ def sync_profile(
no_delete: bool = False,
plan_accumulator: Optional[List[Dict[str, Any]]] = None,
) -> bool:
# SECURITY: Clear cached DNS validations at the start of each sync run.
# This prevents TOCTOU issues where a domain's IP could change between runs.
validate_folder_url.cache_clear()
# SECURITY: Removed cache clearing because we removed the cache on validate_folder_url
# to mitigate TOCTOU (DNS Rebinding) risks. Validation now happens per-fetch.
Comment on lines +1045 to +1046
Copy link

Copilot AI Jan 30, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The comment states "Validation now happens per-fetch" but this is not entirely accurate. Looking at the _fetch_if_valid function below (lines 1058-1060), URLs that are already in the data cache (_cache) are returned directly without re-validation. This means validation does not happen per-fetch for cached URLs, which could still present a TOCTOU risk if DNS records change between syncs within the same process. Consider updating the comment to clarify that validation happens per-fetch for non-cached URLs, or document the accepted risk of using cached data without re-validation.

Copilot uses AI. Check for mistakes.

try:
# Fetch all folder data first
Expand Down
64 changes: 63 additions & 1 deletion tests/test_security.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import os
import stat
import sys
from unittest.mock import MagicMock
import socket
from unittest.mock import MagicMock, patch

import pytest

Expand Down Expand Up @@ -285,3 +286,64 @@
Tests the is_valid_rule function against a strict whitelist of inputs.
"""
assert is_valid_rule(rule) == expected_validity, f"Failed for rule: {rule}"


# Mock helpers for TestValidateFolderUrl
def mock_getaddrinfo_ipv4(ip):

Check warning

Code scanning / Pylint (reported by Codacy)

Missing function docstring Warning test

Missing function docstring

Check warning

Code scanning / Pylint (reported by Codacy)

Argument name "ip" doesn't conform to snake_case naming style Warning test

Argument name "ip" doesn't conform to snake_case naming style

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Argument name "ip" doesn't conform to snake_case naming style Warning test

Argument name "ip" doesn't conform to snake_case naming style

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring
return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (ip, 443))]

def mock_getaddrinfo_ipv6(ip):

Check warning

Code scanning / Pylint (reported by Codacy)

Missing function docstring Warning test

Missing function docstring

Check warning

Code scanning / Pylint (reported by Codacy)

Argument name "ip" doesn't conform to snake_case naming style Warning test

Argument name "ip" doesn't conform to snake_case naming style

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Argument name "ip" doesn't conform to snake_case naming style Warning test

Argument name "ip" doesn't conform to snake_case naming style
return [(socket.AF_INET6, socket.SOCK_STREAM, 6, '', (ip, 443, 0, 0))]

class TestValidateFolderUrl:
"""Security tests for SSRF protection in validate_folder_url."""

def test_rejects_non_https(self):

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring
assert main.validate_folder_url("http://example.com/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
assert main.validate_folder_url("ftp://example.com/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
assert main.validate_folder_url("file:///etc/passwd") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
Comment on lines +301 to +304

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

This test can be made more concise and easier to extend by using pytest.mark.parametrize. This converts multiple assertions into a data-driven test, which is cleaner and provides better failure messages for individual cases.

Suggested change
def test_rejects_non_https(self):
assert main.validate_folder_url("http://example.com/foo") is False
assert main.validate_folder_url("ftp://example.com/foo") is False
assert main.validate_folder_url("file:///etc/passwd") is False
@pytest.mark.parametrize("url", [
"http://example.com/foo",
"ftp://example.com/foo",
"file:///etc/passwd",
])
def test_rejects_non_https(self, url):
assert main.validate_folder_url(url) is False

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

@patch('socket.getaddrinfo')
def test_accepts_valid_public_ip(self, mock_gai):

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring
# Mock domain resolving to public IP (8.8.8.8)
mock_gai.side_effect = lambda host, *args, **kwargs: mock_getaddrinfo_ipv4("8.8.8.8")

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

For a mock that should always return the same value regardless of arguments, using return_value is simpler and more direct than side_effect with a lambda. This improves readability.

Suggested change
mock_gai.side_effect = lambda host, *args, **kwargs: mock_getaddrinfo_ipv4("8.8.8.8")
mock_gai.return_value = mock_getaddrinfo_ipv4("8.8.8.8")

assert main.validate_folder_url("https://example.com/foo") is True

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

def test_rejects_localhost_literal(self):

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function
# Test explicit localhost hostname check
assert main.validate_folder_url("https://localhost/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

@patch('socket.getaddrinfo')
def test_rejects_private_ip_resolution(self, mock_gai):

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring
# Mock domain resolving to private IP (192.168.1.1)
mock_gai.side_effect = lambda host, *args, **kwargs: mock_getaddrinfo_ipv4("192.168.1.1")
assert main.validate_folder_url("https://internal.corp/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

@patch('socket.getaddrinfo')
def test_rejects_loopback_ip_resolution(self, mock_gai):

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring
# Mock domain resolving to loopback IP (127.0.0.1)
mock_gai.side_effect = lambda host, *args, **kwargs: mock_getaddrinfo_ipv4("127.0.0.1")
assert main.validate_folder_url("https://evil.com/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

@patch('socket.getaddrinfo')
def test_rejects_ipv6_loopback_resolution(self, mock_gai):

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring
# Mock domain resolving to IPv6 loopback (::1)
mock_gai.side_effect = lambda host, *args, **kwargs: mock_getaddrinfo_ipv6("::1")
assert main.validate_folder_url("https://ipv6.local/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
Comment on lines +316 to +332

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The tests for private, loopback, and IPv6 loopback resolutions are very similar. They can be combined into a single, parametrized test. This reduces code duplication and makes the test suite easier to maintain and extend with more non-public IP cases in the future. This suggestion also simplifies the mock setup by using return_value instead of a side_effect with a lambda, which is more direct for this use case.

Suggested change
@patch('socket.getaddrinfo')
def test_rejects_private_ip_resolution(self, mock_gai):
# Mock domain resolving to private IP (192.168.1.1)
mock_gai.side_effect = lambda host, *args, **kwargs: mock_getaddrinfo_ipv4("192.168.1.1")
assert main.validate_folder_url("https://internal.corp/foo") is False
@patch('socket.getaddrinfo')
def test_rejects_loopback_ip_resolution(self, mock_gai):
# Mock domain resolving to loopback IP (127.0.0.1)
mock_gai.side_effect = lambda host, *args, **kwargs: mock_getaddrinfo_ipv4("127.0.0.1")
assert main.validate_folder_url("https://evil.com/foo") is False
@patch('socket.getaddrinfo')
def test_rejects_ipv6_loopback_resolution(self, mock_gai):
# Mock domain resolving to IPv6 loopback (::1)
mock_gai.side_effect = lambda host, *args, **kwargs: mock_getaddrinfo_ipv6("::1")
assert main.validate_folder_url("https://ipv6.local/foo") is False
@pytest.mark.parametrize("ip, mock_fn, url", [
("192.168.1.1", mock_getaddrinfo_ipv4, "https://internal.corp/foo"),
("127.0.0.1", mock_getaddrinfo_ipv4, "https://evil.com/foo"),
("::1", mock_getaddrinfo_ipv6, "https://ipv6.local/foo"),
])
@patch('socket.getaddrinfo')
def test_rejects_non_public_ip_resolution(self, mock_gai, ip, mock_fn, url):
"""Tests that domains resolving to private or loopback IPs are rejected."""
mock_gai.return_value = mock_fn(ip)
assert main.validate_folder_url(url) is False

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

def test_rejects_ip_literal_private(self):

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring
# IP literals are checked directly via ipaddress module, bypassing DNS
assert main.validate_folder_url("https://192.168.1.1/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
assert main.validate_folder_url("https://10.0.0.1/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
assert main.validate_folder_url("https://127.0.0.1/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
assert main.validate_folder_url("https://[::1]/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
Comment on lines +334 to +339

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

This test can be made more concise and easier to extend by using pytest.mark.parametrize. This converts multiple assertions into a data-driven test, which is cleaner and provides better failure messages for individual cases.

Suggested change
def test_rejects_ip_literal_private(self):
# IP literals are checked directly via ipaddress module, bypassing DNS
assert main.validate_folder_url("https://192.168.1.1/foo") is False
assert main.validate_folder_url("https://10.0.0.1/foo") is False
assert main.validate_folder_url("https://127.0.0.1/foo") is False
assert main.validate_folder_url("https://[::1]/foo") is False
@pytest.mark.parametrize("url", [
"https://192.168.1.1/foo",
"https://10.0.0.1/foo",
"https://127.0.0.1/foo",
"https://[::1]/foo",
])
def test_rejects_ip_literal_private(self, url):
# IP literals are checked directly via ipaddress module, bypassing DNS
assert main.validate_folder_url(url) is False

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

def test_accepts_ip_literal_public(self):

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function
assert main.validate_folder_url("https://8.8.8.8/foo") is True

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
assert main.validate_folder_url("https://[2001:4860:4860::8888]/foo") is True

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
Comment on lines +341 to +343

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

This test can be made more concise and easier to extend by using pytest.mark.parametrize. This converts multiple assertions into a data-driven test.

Suggested change
def test_accepts_ip_literal_public(self):
assert main.validate_folder_url("https://8.8.8.8/foo") is True
assert main.validate_folder_url("https://[2001:4860:4860::8888]/foo") is True
@pytest.mark.parametrize("url", [
"https://8.8.8.8/foo",
"https://[2001:4860:4860::8888]/foo",
])
def test_accepts_ip_literal_public(self, url):
assert main.validate_folder_url(url) is True

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

@patch('socket.getaddrinfo')
def test_dns_resolution_failure_is_safe(self, mock_gai):

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Method could be a function Warning test

Method could be a function

Check warning

Code scanning / Pylint (reported by Codacy)

Missing method docstring Warning test

Missing method docstring

Check warning

Code scanning / Pylintpython3 (reported by Codacy)

Missing function or method docstring Warning test

Missing function or method docstring

Check warning

Code scanning / Pylint (reported by Codacy)

Method could be a function Warning test

Method could be a function
# Ensure that if DNS fails, we default to False (fail closed)
mock_gai.side_effect = socket.gaierror("Name or service not known")
assert main.validate_folder_url("https://nonexistent.com/foo") is False

Check notice

Code scanning / Bandit

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.

Check notice

Code scanning / Bandit (reported by Codacy)

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code. Note test

Use of assert detected. The enclosed code will be removed when compiling to optimised byte code.
Loading