diff --git a/crates/openshell-bootstrap/src/docker.rs b/crates/openshell-bootstrap/src/docker.rs index 9c365bfe..866bcd5b 100644 --- a/crates/openshell-bootstrap/src/docker.rs +++ b/crates/openshell-bootstrap/src/docker.rs @@ -236,6 +236,62 @@ fn home_dir() -> Option { std::env::var("HOME").ok() } +/// Discover upstream DNS resolvers from systemd-resolved's configuration. +/// +/// Only reads `/run/systemd/resolve/resolv.conf` — the upstream resolver file +/// maintained by systemd-resolved. This file is only present on Linux hosts +/// running systemd-resolved (e.g., Ubuntu), so the function is a no-op on +/// macOS, Windows/WSL, and non-systemd Linux distributions. +/// +/// We intentionally do NOT fall back to `/etc/resolv.conf` here. On Docker +/// Desktop (macOS/Windows), `/etc/resolv.conf` may contain non-loopback +/// resolvers that appear valid but are unreachable via direct UDP from inside +/// the container's network stack. Those environments rely on the entrypoint's +/// iptables DNAT proxy to Docker's embedded DNS — sniffing host resolvers +/// would bypass that proxy and break DNS. +/// +/// Returns an empty vec if no usable resolvers are found. +/// Parse resolv.conf content, extracting nameserver IPs and filtering loopback addresses. +fn parse_resolv_conf(contents: &str) -> Vec { + contents + .lines() + .filter_map(|line| { + let line = line.trim(); + if !line.starts_with("nameserver") { + return None; + } + let ip = line.split_whitespace().nth(1)?; + if ip.starts_with("127.") || ip == "::1" { + return None; + } + Some(ip.to_string()) + }) + .collect() +} + +fn resolve_upstream_dns() -> Vec { + let paths = ["/run/systemd/resolve/resolv.conf"]; + + for path in &paths { + if let Ok(contents) = std::fs::read_to_string(path) { + let resolvers = parse_resolv_conf(&contents); + + if !resolvers.is_empty() { + tracing::debug!( + "Discovered {} upstream DNS resolver(s) from {}: {}", + resolvers.len(), + path, + resolvers.join(", "), + ); + return resolvers; + } + } + } + + tracing::debug!("No upstream DNS resolvers found in host resolver config"); + Vec::new() +} + /// Create an SSH Docker client from remote options. pub async fn create_ssh_docker_client(remote: &RemoteOptions) -> Result { // Ensure destination has ssh:// prefix @@ -455,6 +511,7 @@ pub async fn ensure_container( registry_username: Option<&str>, registry_token: Option<&str>, gpu: bool, + is_remote: bool, ) -> Result<()> { let container_name = container_name(name); @@ -675,6 +732,17 @@ pub async fn ensure_container( env_vars.push("GPU_ENABLED=true".to_string()); } + // Pass upstream DNS resolvers discovered on the host so the entrypoint + // can configure k3s without probing files inside the container. + // Skip for remote deploys — the local host's resolvers are likely wrong + // for the remote Docker host (different network, split-horizon DNS, etc.). + if !is_remote { + let upstream_dns = resolve_upstream_dns(); + if !upstream_dns.is_empty() { + env_vars.push(format!("UPSTREAM_DNS={}", upstream_dns.join(","))); + } + } + let env = Some(env_vars); let config = ContainerCreateBody { @@ -1195,4 +1263,93 @@ mod tests { "should return a reasonable number of sockets" ); } + + #[test] + fn resolve_upstream_dns_filters_loopback() { + // This test validates the function runs without panic on the current host. + // The exact output depends on the host's DNS config, but loopback + // addresses must never appear in the result. + let resolvers = resolve_upstream_dns(); + for r in &resolvers { + assert!( + !r.starts_with("127."), + "IPv4 loopback should be filtered: {r}" + ); + assert_ne!(r, "::1", "IPv6 loopback should be filtered"); + } + } + + #[test] + fn resolve_upstream_dns_returns_vec() { + // Verify the function returns a vec (may be empty in some CI environments + // where no resolv.conf exists, but should never panic). + let resolvers = resolve_upstream_dns(); + assert!( + resolvers.len() <= 20, + "should return a reasonable number of resolvers" + ); + } + + #[test] + fn parse_resolv_conf_filters_ipv4_loopback() { + let input = "nameserver 127.0.0.1\nnameserver 127.0.0.53\nnameserver 127.0.0.11\n"; + assert!(parse_resolv_conf(input).is_empty()); + } + + #[test] + fn parse_resolv_conf_filters_ipv6_loopback() { + let input = "nameserver ::1\n"; + assert!(parse_resolv_conf(input).is_empty()); + } + + #[test] + fn parse_resolv_conf_passes_real_resolvers() { + let input = "nameserver 8.8.8.8\nnameserver 1.1.1.1\n"; + assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]); + } + + #[test] + fn parse_resolv_conf_mixed_loopback_and_real() { + let input = + "nameserver 127.0.0.53\nnameserver ::1\nnameserver 10.0.0.1\nnameserver 172.16.0.1\n"; + assert_eq!(parse_resolv_conf(input), vec!["10.0.0.1", "172.16.0.1"]); + } + + #[test] + fn parse_resolv_conf_ignores_comments_and_other_lines() { + let input = + "# nameserver 8.8.8.8\nsearch example.com\noptions ndots:5\nnameserver 1.1.1.1\n"; + assert_eq!(parse_resolv_conf(input), vec!["1.1.1.1"]); + } + + #[test] + fn parse_resolv_conf_handles_tabs_and_extra_spaces() { + let input = "nameserver\t8.8.8.8\nnameserver 1.1.1.1\n"; + assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]); + } + + #[test] + fn parse_resolv_conf_empty_input() { + assert!(parse_resolv_conf("").is_empty()); + assert!(parse_resolv_conf(" \n\n").is_empty()); + } + + #[test] + fn parse_resolv_conf_bare_nameserver_keyword() { + assert!(parse_resolv_conf("nameserver\n").is_empty()); + assert!(parse_resolv_conf("nameserver \n").is_empty()); + } + + #[test] + fn parse_resolv_conf_systemd_resolved_typical() { + let input = + "# This is /run/systemd/resolve/resolv.conf\nnameserver 192.168.1.1\nsearch lan\n"; + assert_eq!(parse_resolv_conf(input), vec!["192.168.1.1"]); + } + + #[test] + fn parse_resolv_conf_crlf_line_endings() { + let input = "nameserver 8.8.8.8\r\nnameserver 1.1.1.1\r\n"; + assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]); + } } diff --git a/crates/openshell-bootstrap/src/lib.rs b/crates/openshell-bootstrap/src/lib.rs index 9098fd4a..8e9e2e74 100644 --- a/crates/openshell-bootstrap/src/lib.rs +++ b/crates/openshell-bootstrap/src/lib.rs @@ -417,6 +417,7 @@ where registry_username.as_deref(), registry_token.as_deref(), gpu, + remote_opts.is_some(), ) .await?; start_container(&target_docker, &name).await?; diff --git a/deploy/docker/cluster-entrypoint.sh b/deploy/docker/cluster-entrypoint.sh index 84b8cf9a..0e75e67c 100644 --- a/deploy/docker/cluster-entrypoint.sh +++ b/deploy/docker/cluster-entrypoint.sh @@ -69,7 +69,46 @@ wait_for_default_route() { # 3. Adding DNAT rules so traffic to :53 reaches Docker's DNS # 4. Writing that IP into the k3s resolv.conf +# Extract upstream DNS resolvers reachable from k3s pod namespaces. +# Docker's embedded DNS (127.0.0.11) is namespace-local — DNAT to it from +# pod traffic is dropped as a martian packet. Use real upstream servers instead. +# +# Priority: +# 1. UPSTREAM_DNS env var (set by bootstrap, comma-separated) +# 2. /etc/resolv.conf (fallback for non-bootstrap launches) +get_upstream_resolvers() { + local resolvers="" + + # Bootstrap-provided resolvers (sniffed from host by the Rust bootstrap crate) + if [ -n "${UPSTREAM_DNS:-}" ]; then + resolvers=$(printf '%s\n' "$UPSTREAM_DNS" | tr ',' '\n' | \ + awk '{ip=$1; if(ip !~ /^127\./ && ip != "::1" && ip != "") print ip}') + fi + + # Fallback: Docker-generated resolv.conf may have non-loopback servers + if [ -z "$resolvers" ]; then + resolvers=$(awk '/^nameserver/{ip=$2; gsub(/\r/,"",ip); if(ip !~ /^127\./ && ip != "::1") print ip}' \ + /etc/resolv.conf) + fi + + echo "$resolvers" +} + setup_dns_proxy() { + # Prefer upstream resolvers that work across network namespaces. + # This avoids the DNAT-to-loopback problem on systemd-resolved hosts. + UPSTREAM_DNS=$(get_upstream_resolvers) + if [ -n "$UPSTREAM_DNS" ]; then + : > "$RESOLV_CONF" + echo "$UPSTREAM_DNS" | while read -r ns; do + [ -n "$ns" ] && echo "nameserver $ns" >> "$RESOLV_CONF" + done + echo "DNS: using upstream resolvers directly (avoids cross-namespace DNAT)" + cat "$RESOLV_CONF" + return 0 + fi + + # Fall back to DNAT proxy when no upstream resolvers are available. # Extract Docker's actual DNS listener ports from the DOCKER_OUTPUT chain. # Docker sets up rules like: # -A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp --dport 53 -j DNAT --to-destination 127.0.0.11: @@ -160,6 +199,8 @@ verify_dns() { sleep 1 i=$((i + 1)) done + echo "Warning: DNS verification failed for $lookup_host after $attempts attempts" + echo " resolv.conf: $(head -3 "$RESOLV_CONF" 2>/dev/null)" return 1 }