diff --git a/Cargo.lock b/Cargo.lock index ee28f52..79d78c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,7 +99,7 @@ checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "blasthttp" -version = "0.7.0" +version = "0.8.0" dependencies = [ "brotli", "bytes", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -385,9 +385,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -424,9 +424,9 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -766,9 +766,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", diff --git a/Cargo.toml b/Cargo.toml index 3a7df73..11d73b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "blasthttp" -version = "0.7.0" +version = "0.8.0" edition = "2024" description = "Offensive-first HTTP library with Python bindings" license = "GPL-3.0" diff --git a/README.md b/README.md index 2dac3b3..84501ac 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ blasthttp https://example.com -L # Through a proxy blasthttp https://example.com -x http://127.0.0.1:8080 +# ...but connect directly to certain hosts (host, *.suffix, IP, or CIDR) +blasthttp https://example.com -x http://127.0.0.1:8080 --no-proxy '*.internal.corp' --no-proxy 10.0.0.0/8 + # Force specific TLS versions/ciphers blasthttp https://legacy-server.com --min-tls 1.0 --ciphers "RC4-SHA" @@ -106,6 +109,7 @@ Output is JSON (one object per response), including status, headers, redirect ch | `--max-body-size` | Max response body (bytes) | 10 MB | | `--verify` | Enable TLS cert validation | off | | `-x, --proxy` | HTTP/SOCKS proxy URL | | +| `--no-proxy` | Host that bypasses the proxy: `host`, `*.suffix`, IP, CIDR, or `*` (repeatable) | | | `--ciphers` | OpenSSL cipher string | all | | `--min-tls` | Minimum TLS version (1.0–1.3) | | | `--max-tls` | Maximum TLS version (1.0–1.3) | | @@ -232,6 +236,31 @@ response = await client.request( ) ``` +### Proxy exclusions (`no_proxy`) + +`proxy` routes a request through an HTTP or SOCKS5 proxy; `no_proxy` is a per-request list of hosts that bypass it and connect directly — the `NO_PROXY` equivalent. It's accepted by `request()`, `download()`, `raw_connect()`, and `BatchConfig`, and as the repeatable `--no-proxy` CLI flag. + +```python +# Proxy everything except internal hosts and the loopback range. +r = await client.request( + "https://example.com/", + proxy="http://127.0.0.1:8080", + no_proxy=["localhost", "*.internal.corp", "10.0.0.0/8"], +) +``` + +Each entry matches case-insensitively as: + +- an exact hostname (`elastic.corp`); +- a domain suffix — `*.corp`, `.corp`, and `corp` are equivalent and match the domain itself plus any subdomain; +- a single IP (`127.0.0.1`, `::1`); +- a CIDR range (`10.0.0.0/8`, `fd00::/8`), matched only against IP hosts; +- or `*` to bypass the proxy for every host. + +The proxy/`no_proxy` decision is re-evaluated on **every redirect hop**, not just the first URL: if a request starts on an excluded (direct) host and is redirected onto a non-excluded host, the later hop goes through the proxy — and vice versa. + +`no_proxy` only does anything when a `proxy` is set, so passing it without one is rejected up front (the request errors rather than silently ignoring the exclusions). + ### Global Rate Limiting Set a client-level rate limit (requests per second) that applies to **all** request methods — `request()`, `request_batch()`, `request_batch_stream()`, and `download()`: @@ -290,7 +319,7 @@ async def main(): asyncio.run(main()) ``` -`raw_connect()` takes all the same TLS knobs as `request()` — `verify_certs`, `cipher_string`, `min_tls_version`, `max_tls_version`, `resolve_ip`, `proxy`. `alpn_protocols` is a list of byte-strings (commonly `["h2", "http/1.1"]`) used in the TLS ALPN extension. After the handshake, `conn.negotiated_alpn` reports which one the server picked (or `None` if no ALPN was negotiated, including all plain-HTTP connections). +`raw_connect()` takes all the same TLS knobs as `request()` — `verify_certs`, `cipher_string`, `min_tls_version`, `max_tls_version`, `resolve_ip`, `proxy`, `no_proxy`. `alpn_protocols` is a list of byte-strings (commonly `["h2", "http/1.1"]`) used in the TLS ALPN extension. After the handshake, `conn.negotiated_alpn` reports which one the server picked (or `None` if no ALPN was negotiated, including all plain-HTTP connections). `conn.peer_ip` returns the actual IP address the OS connected to, same as `Response.peer_ip`. `None` when the connection went through a proxy. diff --git a/pyproject.toml b/pyproject.toml index 2002f3e..d2dcc0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "blasthttp" -version = "0.7.0" +version = "0.8.0" description = "Offensive-first HTTP library" license = "GPL-3.0" requires-python = ">=3.10" @@ -20,7 +20,7 @@ managed = false dev = [ "pytest>=8", "pytest-asyncio>=0.23", - "ruff==0.15.13", + "ruff==0.15.15", ] [tool.pytest.ini_options] diff --git a/src/client/hyper.rs b/src/client/hyper.rs index 43815ce..f189dc3 100644 --- a/src/client/hyper.rs +++ b/src/client/hyper.rs @@ -532,7 +532,7 @@ impl HyperClient { /// Determine the connection mode for a given request config + target URI. fn conn_mode(config: &RequestConfig, target_uri: &http::Uri) -> Result { - match config.proxy.as_deref() { + match config.effective_proxy(target_uri.host().unwrap_or("")) { None => Ok(ConnMode::Direct), Some(proxy_url) => { let proxy_uri: http::Uri = @@ -684,6 +684,8 @@ pub(crate) async fn connect_stream( > { use super::proxy::{self, ProxyScheme}; + config.validate_proxy().map_err(ClientError::other)?; + let v = config.verbosity; let host = target_uri.host().unwrap_or("").to_string(); let is_https = target_uri.scheme_str() == Some("https"); @@ -694,7 +696,7 @@ pub(crate) async fn connect_stream( // tunnel from there. resolve_ip is ignored when a proxy is set — DNS // resolution happens at the proxy for SOCKS5, and CONNECT addresses the // target by hostname. - let proxy_config = match config.proxy.as_ref() { + let proxy_config = match config.effective_proxy(&host) { Some(p) => Some(proxy::parse_proxy_url(p)?), None => None, }; @@ -1183,6 +1185,7 @@ fn retry_backoff(attempt: u32, min_wait: Duration, max_wait: Duration) -> Durati impl HttpClient for HyperClient { async fn send(&self, config: &RequestConfig) -> Result { + config.validate_proxy().map_err(ClientError::other)?; let timeout_duration = Duration::from_secs(config.timeout()); let max_retries = config.max_retries(); let min_wait = config.retry_wait_min(); @@ -1267,7 +1270,7 @@ impl HyperClient { })?; debug_record(log, v, 1, &format!("-> {} {}", config.method(), uri)); - if let Some(ref proxy) = config.proxy { + if let Some(proxy) = config.effective_proxy(uri.host().unwrap_or("")) { debug_record(log, v, 1, &format!(" Proxy: {}", proxy)); } if !config.should_verify_certs() { @@ -1321,41 +1324,37 @@ impl HyperClient { }); } - let mode = Self::conn_mode(config, &uri)?; - - // Forward proxy: dispatch directly via TCP + http1::SendRequest - // (bypasses hyper Client's URI normalization to preserve absolute-form) - let is_forward_proxy = matches!(&mode, ConnMode::ForwardProxy(_)); - let proxy_url_for_fwd = if let ConnMode::ForwardProxy(ref url) = mode { - Some(url.clone()) - } else { - None - }; - - // For non-forward-proxy modes, get the cached hyper Client - let cached = if is_forward_proxy { - None - } else { - Some(self.get_or_build(config, &mode)?) - }; - let mut redirect_chain: Vec = Vec::new(); let mut hops = 0u32; - // Per-hop peer IP lookup. Returns None when: - // • the request went through a proxy (the connector recorded - // the proxy IP under the proxy's authority, so the target's - // authority isn't in the map) - // • forward-proxy dispatch (bypasses the connector entirely) - // • peer_addr() failed at connect time (vanishingly rare) - let lookup_peer_ip = |target_uri: &http::Uri| -> Option { - let key = peer_slot_key(target_uri)?; - let cached = cached.as_ref()?; - let map = cached.peer_slot.lock().ok()?; - map.get(&key).map(|ip| ip.to_string()) - }; - loop { + // Decide the connection mode for the *current* target host on every + // hop, not just the first. A redirect can send the request to a + // different host, and the proxy / no_proxy decision has to follow + // it. Freezing the first hop's choice would otherwise let a request + // that started direct keep connecting directly after a redirect onto + // a proxied host (leaking traffic past the proxy), and let a request + // that started proxied keep using the proxy after a redirect onto a + // no_proxy host. Clients are cached by mode, so hops that share a + // mode reuse the same client. + let mode = Self::conn_mode(config, &uri)?; + + // Forward proxy: dispatch directly via TCP + http1::SendRequest + // (bypasses hyper Client's URI normalization to preserve absolute-form) + let is_forward_proxy = matches!(&mode, ConnMode::ForwardProxy(_)); + let proxy_url_for_fwd = if let ConnMode::ForwardProxy(ref url) = mode { + Some(url.clone()) + } else { + None + }; + + // For non-forward-proxy modes, get the cached hyper Client. + let cached = if is_forward_proxy { + None + } else { + Some(self.get_or_build(config, &mode)?) + }; + let resp = if let Some(ref proxy_url) = proxy_url_for_fwd { dispatch_forward_proxy(proxy_url, &uri, config, log).await? } else { @@ -1364,7 +1363,17 @@ impl HyperClient { let hop_ms = start.elapsed().as_millis(); debug_record(log, v, 1, &format!("<- {} ({}ms)", resp.status, hop_ms)); - let hop_peer_ip = lookup_peer_ip(&uri); + // Per-hop peer IP lookup. Returns None when: + // • the request went through a proxy (the connector recorded + // the proxy IP under the proxy's authority, so the target's + // authority isn't in the map) + // • forward-proxy dispatch (bypasses the connector entirely) + // • peer_addr() failed at connect time (vanishingly rare) + let hop_peer_ip = peer_slot_key(&uri).and_then(|key| { + let cached = cached.as_ref()?; + let map = cached.peer_slot.lock().ok()?; + map.get(&key).map(|ip| ip.to_string()) + }); if is_redirect(resp.status) && config.should_follow_redirects() { hops += 1; diff --git a/src/config.rs b/src/config.rs index 1966fa2..1c381d5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,6 +12,11 @@ pub struct RequestConfig { pub max_redirects: Option, pub verify_certs: Option, pub proxy: Option, + /// Hosts that bypass `proxy` and connect directly (NO_PROXY equivalent). + /// Entries may be exact hostnames, domain suffixes (`*.corp` / `.corp` / + /// `corp`), single IPs, CIDR ranges (`10.0.0.0/8`), or `*` for all. + #[serde(default)] + pub no_proxy: Vec, pub cipher_string: Option, pub min_tls_version: Option, pub max_tls_version: Option, @@ -52,6 +57,7 @@ impl RequestConfig { max_redirects: None, verify_certs: None, proxy: None, + no_proxy: Vec::new(), cipher_string: None, min_tls_version: None, max_tls_version: None, @@ -90,6 +96,33 @@ impl RequestConfig { self.verify_certs.unwrap_or(false) } + /// The proxy URL to use for a request to `target_host`, or `None` when no + /// proxy is configured or the host matches a `no_proxy` entry. + pub fn effective_proxy(&self, target_host: &str) -> Option<&str> { + let proxy = self.proxy.as_deref()?; + if host_bypasses_proxy(target_host, &self.no_proxy) { + None + } else { + Some(proxy) + } + } + + /// `no_proxy` only has an effect alongside a `proxy` — it lists hosts that + /// bypass that proxy. Setting it without a proxy is silently a no-op and + /// almost always a mistake, so reject it up front. Returns the error + /// message (caller wraps it in its error type). + pub fn validate_proxy(&self) -> Result<(), String> { + let proxy_set = self.proxy.as_deref().is_some_and(|p| !p.trim().is_empty()); + if !self.no_proxy.is_empty() && !proxy_set { + return Err( + "no_proxy is set but no proxy is configured; no_proxy only has an effect \ + when a proxy is also set" + .to_string(), + ); + } + Ok(()) + } + pub fn max_retries(&self) -> u32 { self.retries.unwrap_or(1) } @@ -102,3 +135,241 @@ impl RequestConfig { std::time::Duration::from_millis(self.retry_wait_max_ms.unwrap_or(30000)) } } + +use std::net::IpAddr; + +/// Returns true if `host` matches any `no_proxy` pattern, meaning a request to +/// it should bypass the configured proxy. Matching follows the conventional +/// NO_PROXY rules: +/// +/// - `*` matches every host. +/// - A CIDR (`10.0.0.0/8`, `fd00::/8`) matches when `host` is an IP inside it. +/// - A bare IP matches that exact address. +/// - Anything else is treated as a domain: it matches the domain itself and any +/// subdomain, case-insensitively. Leading `*.` or `.` are accepted and +/// ignored (`*.corp`, `.corp`, and `corp` are equivalent). +pub(crate) fn host_bypasses_proxy(host: &str, patterns: &[String]) -> bool { + if host.is_empty() { + return false; + } + // Strip brackets from IPv6 literals so "[::1]" parses as an address. + let host = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(host); + let host_ip = host.parse::().ok(); + let host = host.trim_end_matches('.'); + + for pattern in patterns { + let pattern = pattern.trim(); + if pattern.is_empty() { + continue; + } + if pattern == "*" { + return true; + } + + if let Some((net, prefix)) = pattern.split_once('/') { + // CIDR: only meaningful when the target is an IP. + if let (Some(ip), Ok(net_ip), Ok(prefix_len)) = ( + host_ip, + net.trim().parse::(), + prefix.trim().parse::(), + ) && ip_in_cidr(ip, net_ip, prefix_len) + { + return true; + } + continue; + } + + if let Ok(pattern_ip) = pattern.parse::() { + if Some(pattern_ip) == host_ip { + return true; + } + continue; + } + + // Hostname / domain-suffix match. An IP target never matches a name. + if host_ip.is_some() { + continue; + } + let suffix = pattern.trim_start_matches('*').trim_matches('.'); + if !suffix.is_empty() && host_matches_suffix(host, suffix) { + return true; + } + } + false +} + +/// Case-insensitive check that `host` equals `suffix` or is a subdomain of it +/// (i.e. `host` ends in `.`). Operates on bytes to avoid allocating. +fn host_matches_suffix(host: &str, suffix: &str) -> bool { + let (host, suffix) = (host.as_bytes(), suffix.as_bytes()); + if host.len() == suffix.len() { + return host.eq_ignore_ascii_case(suffix); + } + match host.len().checked_sub(suffix.len()) { + Some(start) if start >= 1 && host[start - 1] == b'.' => { + host[start..].eq_ignore_ascii_case(suffix) + } + _ => false, + } +} + +/// True if `ip` falls within the `net`/`prefix_len` CIDR block. Mismatched +/// address families (v4 vs v6) never match. +fn ip_in_cidr(ip: IpAddr, net: IpAddr, prefix_len: u8) -> bool { + match (ip, net) { + (IpAddr::V4(ip), IpAddr::V4(net)) => { + if prefix_len > 32 { + return false; + } + if prefix_len == 0 { + return true; + } + let mask = u32::MAX << (32 - prefix_len); + (u32::from(ip) & mask) == (u32::from(net) & mask) + } + (IpAddr::V6(ip), IpAddr::V6(net)) => { + if prefix_len > 128 { + return false; + } + if prefix_len == 0 { + return true; + } + let mask = u128::MAX << (128 - prefix_len); + (u128::from(ip) & mask) == (u128::from(net) & mask) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pats(v: &[&str]) -> Vec { + v.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn exact_hostname() { + assert!(host_bypasses_proxy("localhost", &pats(&["localhost"]))); + assert!(host_bypasses_proxy( + "elastic.corp", + &pats(&["elastic.corp"]) + )); + assert!(!host_bypasses_proxy( + "example.com", + &pats(&["elastic.corp"]) + )); + } + + #[test] + fn case_insensitive_and_trailing_dot() { + assert!(host_bypasses_proxy("LocalHost.", &pats(&["localhost"]))); + assert!(host_bypasses_proxy( + "API.Internal.Corp", + &pats(&["*.internal.corp"]) + )); + } + + #[test] + fn domain_suffix_forms_are_equivalent() { + for p in ["*.internal.corp", ".internal.corp", "internal.corp"] { + assert!( + host_bypasses_proxy("api.internal.corp", &pats(&[p])), + "subdomain should match {p}" + ); + assert!( + host_bypasses_proxy("internal.corp", &pats(&[p])), + "apex should match {p}" + ); + } + // A suffix must align on a label boundary. + assert!(!host_bypasses_proxy( + "notinternal.corp", + &pats(&["*.internal.corp"]) + )); + assert!(!host_bypasses_proxy( + "internal.corp.evil.com", + &pats(&["internal.corp"]) + )); + } + + #[test] + fn bare_ip() { + assert!(host_bypasses_proxy("127.0.0.1", &pats(&["127.0.0.1"]))); + assert!(!host_bypasses_proxy("127.0.0.2", &pats(&["127.0.0.1"]))); + assert!(host_bypasses_proxy("::1", &pats(&["::1"]))); + assert!(host_bypasses_proxy("[::1]", &pats(&["::1"]))); + } + + #[test] + fn cidr_v4() { + let p = pats(&["10.0.0.0/8"]); + assert!(host_bypasses_proxy("10.1.2.3", &p)); + assert!(host_bypasses_proxy("10.255.255.255", &p)); + assert!(!host_bypasses_proxy("11.0.0.1", &p)); + // A hostname is never inside a CIDR. + assert!(!host_bypasses_proxy("ten.example.com", &p)); + } + + #[test] + fn cidr_v6_and_zero_prefix() { + assert!(host_bypasses_proxy("fd00::1", &pats(&["fd00::/8"]))); + assert!(!host_bypasses_proxy("fe00::1", &pats(&["fd00::/8"]))); + assert!(host_bypasses_proxy("8.8.8.8", &pats(&["0.0.0.0/0"]))); + } + + #[test] + fn family_mismatch_never_matches() { + assert!(!host_bypasses_proxy("10.0.0.1", &pats(&["fd00::/8"]))); + assert!(!host_bypasses_proxy("fd00::1", &pats(&["10.0.0.0/8"]))); + } + + #[test] + fn wildcard_and_empty() { + assert!(host_bypasses_proxy("anything.com", &pats(&["*"]))); + assert!(!host_bypasses_proxy("anything.com", &pats(&[]))); + assert!(!host_bypasses_proxy("", &pats(&["*"]))); + assert!(!host_bypasses_proxy("host", &pats(&["", " "]))); + } + + #[test] + fn effective_proxy_respects_exclusions() { + let mut cfg = RequestConfig::new("http://x/".into()); + cfg.proxy = Some("http://proxy:8080".into()); + cfg.no_proxy = pats(&["127.0.0.1", "*.internal.corp"]); + assert_eq!( + cfg.effective_proxy("example.com"), + Some("http://proxy:8080") + ); + assert_eq!(cfg.effective_proxy("127.0.0.1"), None); + assert_eq!(cfg.effective_proxy("api.internal.corp"), None); + // No proxy configured -> always None regardless of no_proxy. + cfg.proxy = None; + assert_eq!(cfg.effective_proxy("example.com"), None); + } + + #[test] + fn validate_proxy_rejects_no_proxy_without_proxy() { + let mut cfg = RequestConfig::new("http://x/".into()); + + // Neither set, or only proxy set -> fine. + assert!(cfg.validate_proxy().is_ok()); + cfg.proxy = Some("http://proxy:8080".into()); + assert!(cfg.validate_proxy().is_ok()); + + // no_proxy alongside a proxy -> fine. + cfg.no_proxy = pats(&["127.0.0.1"]); + assert!(cfg.validate_proxy().is_ok()); + + // no_proxy without a proxy -> error. + cfg.proxy = None; + assert!(cfg.validate_proxy().is_err()); + // An empty/whitespace proxy counts as unset. + cfg.proxy = Some(" ".into()); + assert!(cfg.validate_proxy().is_err()); + } +} diff --git a/src/main.rs b/src/main.rs index 38b171e..24a5219 100644 --- a/src/main.rs +++ b/src/main.rs @@ -60,6 +60,11 @@ struct Cli { #[arg(short = 'x', long)] proxy: Option, + /// Hosts that bypass the proxy (repeatable). Accepts hostnames, domain + /// suffixes (*.corp), IPs, or CIDRs (10.0.0.0/8). + #[arg(long = "no-proxy", num_args = 1)] + no_proxy: Vec, + /// OpenSSL cipher string (e.g. "ALL", "HIGH", "RC4-SHA") #[arg(long)] ciphers: Option, @@ -109,6 +114,7 @@ fn build_config(cli: &Cli, url: String) -> RequestConfig { config.timeout_seconds = cli.timeout; config.max_body_size = cli.max_body_size; config.proxy = cli.proxy.clone(); + config.no_proxy = cli.no_proxy.clone(); config.cipher_string = cli.ciphers.clone(); config.min_tls_version = cli.min_tls.clone(); config.max_tls_version = cli.max_tls.clone(); diff --git a/src/python.rs b/src/python.rs index cb0ac5a..2a95a0e 100644 --- a/src/python.rs +++ b/src/python.rs @@ -871,6 +871,7 @@ impl BlastHTTP { max_redirects=None, verify_certs=None, proxy=None, + no_proxy=None, cipher_string=None, min_tls_version=None, max_tls_version=None, @@ -896,6 +897,7 @@ impl BlastHTTP { max_redirects: Option, verify_certs: Option, proxy: Option, + no_proxy: Option>, cipher_string: Option, min_tls_version: Option, max_tls_version: Option, @@ -919,6 +921,7 @@ impl BlastHTTP { max_redirects, verify_certs, proxy, + no_proxy: no_proxy.unwrap_or_default(), cipher_string, min_tls_version, max_tls_version, @@ -1079,6 +1082,7 @@ impl BlastHTTP { timeout=None, verify_certs=None, proxy=None, + no_proxy=None, headers=None, retries=None, ))] @@ -1092,6 +1096,7 @@ impl BlastHTTP { timeout: Option, verify_certs: Option, proxy: Option, + no_proxy: Option>, headers: Option>, retries: Option, ) -> PyResult> { @@ -1106,6 +1111,7 @@ impl BlastHTTP { max_redirects: Some(10), verify_certs, proxy, + no_proxy: no_proxy.unwrap_or_default(), cipher_string: None, min_tls_version: None, max_tls_version: None, @@ -1163,6 +1169,7 @@ impl BlastHTTP { max_tls_version=None, resolve_ip=None, proxy=None, + no_proxy=None, alpn_protocols=None, ))] #[allow(clippy::too_many_arguments)] @@ -1176,6 +1183,7 @@ impl BlastHTTP { max_tls_version: Option, resolve_ip: Option, proxy: Option, + no_proxy: Option>, alpn_protocols: Option>, ) -> PyResult> { let mut config = RequestConfig::new(url.clone()); @@ -1185,6 +1193,7 @@ impl BlastHTTP { config.max_tls_version = max_tls_version; config.resolve_ip = resolve_ip; config.proxy = proxy; + config.no_proxy = no_proxy.unwrap_or_default(); config.alpn_protocols = alpn_protocols; let limiter = self.rate_limiter.clone(); @@ -1330,6 +1339,8 @@ struct PyBatchConfig { #[pyo3(get, set)] proxy: Option, #[pyo3(get, set)] + no_proxy: Option>, + #[pyo3(get, set)] cipher_string: Option, #[pyo3(get, set)] min_tls_version: Option, @@ -1363,6 +1374,7 @@ impl PyBatchConfig { max_redirects=None, verify_certs=None, proxy=None, + no_proxy=None, cipher_string=None, min_tls_version=None, max_tls_version=None, @@ -1385,6 +1397,7 @@ impl PyBatchConfig { max_redirects: Option, verify_certs: Option, proxy: Option, + no_proxy: Option>, cipher_string: Option, min_tls_version: Option, max_tls_version: Option, @@ -1406,6 +1419,7 @@ impl PyBatchConfig { max_redirects, verify_certs, proxy, + no_proxy, cipher_string, min_tls_version, max_tls_version, @@ -1432,6 +1446,7 @@ impl Clone for PyBatchConfig { max_redirects: self.max_redirects, verify_certs: self.verify_certs, proxy: self.proxy.clone(), + no_proxy: self.no_proxy.clone(), cipher_string: self.cipher_string.clone(), min_tls_version: self.min_tls_version.clone(), max_tls_version: self.max_tls_version.clone(), @@ -1461,6 +1476,7 @@ impl PyBatchConfig { max_redirects: self.max_redirects, verify_certs: self.verify_certs, proxy: self.proxy, + no_proxy: self.no_proxy.unwrap_or_default(), cipher_string: self.cipher_string, min_tls_version: self.min_tls_version, max_tls_version: self.max_tls_version, diff --git a/tests/cli_no_proxy.rs b/tests/cli_no_proxy.rs new file mode 100644 index 0000000..b9d37b2 --- /dev/null +++ b/tests/cli_no_proxy.rs @@ -0,0 +1,252 @@ +// Integration tests for the `--no-proxy` CLI flag. +// +// These spawn the compiled `blasthttp` binary against a local origin server +// and a hit-counting HTTP proxy, then assert (via the response body and the +// proxy's hit counter) whether each request went through the proxy or +// connected directly. This is the one no_proxy surface the library-level +// tests don't exercise: the CLI arg -> build_config -> effective_proxy path. + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::process::Command; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread; +use std::time::Duration; + +/// Listener core: bind a loopback socket on `bind_host`, then for every +/// accepted connection bump `counter` (if set), drain the request head, and +/// write the fixed `response`. The thread is detached and lives for the rest of +/// the test process. Returns the bound port. `counter` is how we tell whether a +/// given host was actually connected to. +fn spawn_raw(bind_host: &str, response: String, counter: Option>) -> u16 { + let listener = TcpListener::bind(format!("{bind_host}:0")).unwrap(); + let port = listener.local_addr().unwrap().port(); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + if let Some(ref c) = counter { + c.fetch_add(1, Ordering::SeqCst); + } + stream.set_read_timeout(Some(Duration::from_secs(5))).ok(); + // Drain the request head so the client isn't writing into a + // half-closed socket when we reply. + let mut buf = Vec::new(); + let mut tmp = [0u8; 512]; + while let Ok(n) = stream.read(&mut tmp) { + if n == 0 { + break; + } + buf.extend_from_slice(&tmp[..n]); + if buf.windows(4).any(|w| w == b"\r\n\r\n") || buf.len() > 16384 { + break; + } + } + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + } + }); + port +} + +/// HTTP/1.1 server on `bind_host` that answers every request with a 200 + `body`. +fn spawn_server(bind_host: &str, body: &str, counter: Option>) -> u16 { + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + spawn_raw(bind_host, response, counter) +} + +/// HTTP/1.1 server on `bind_host` that 302-redirects every request to `location`. +fn spawn_redirect_server( + bind_host: &str, + location: &str, + counter: Option>, +) -> u16 { + let response = format!( + "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + spawn_raw(bind_host, response, counter) +} + +/// Run the CLI against `url` through `proxy` with the given `--no-proxy` +/// entries, returning combined stdout+stderr. When `follow`, pass `-L` so +/// redirects are followed (each hop re-evaluates the proxy decision). +fn run_cli(url: &str, proxy: &str, no_proxy: &[&str], follow: bool) -> String { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_blasthttp")); + cmd.arg(url).arg("-x").arg(proxy); + for entry in no_proxy { + cmd.arg("--no-proxy").arg(entry); + } + if follow { + cmd.arg("-L"); + } + cmd.arg("-vv"); // include the response body in the JSON output + let output = cmd.output().expect("failed to run blasthttp binary"); + let mut combined = String::from_utf8_lossy(&output.stdout).into_owned(); + combined.push_str(&String::from_utf8_lossy(&output.stderr)); + combined +} + +/// Fresh origin + proxy per case so the hit counter is isolated. When +/// `expect_direct`, the request must reach the origin (`TARGET_DIRECT`) and +/// never touch the proxy; otherwise it must go through the proxy. +fn assert_routing(no_proxy: &[&str], expect_direct: bool) { + let proxy_hits = Arc::new(AtomicUsize::new(0)); + let target_port = spawn_server("127.0.0.1", "TARGET_DIRECT", None); + let proxy_port = spawn_server("127.0.0.1", "VIA_PROXY", Some(proxy_hits.clone())); + + let url = format!("http://127.0.0.1:{target_port}/foo"); + let proxy = format!("http://127.0.0.1:{proxy_port}"); + let out = run_cli(&url, &proxy, no_proxy, false); + + if expect_direct { + assert!( + out.contains("TARGET_DIRECT"), + "expected direct connection, got: {out}" + ); + assert_eq!( + proxy_hits.load(Ordering::SeqCst), + 0, + "proxy should have been bypassed" + ); + } else { + assert!( + out.contains("VIA_PROXY"), + "expected proxied connection, got: {out}" + ); + assert_eq!( + proxy_hits.load(Ordering::SeqCst), + 1, + "proxy should have been used once" + ); + } +} + +#[test] +fn no_exclusion_uses_proxy() { + assert_routing(&[], false); +} + +#[test] +fn exact_ip_bypasses_proxy() { + assert_routing(&["127.0.0.1"], true); +} + +#[test] +fn non_matching_cidr_uses_proxy() { + assert_routing(&["10.0.0.0/8"], false); +} + +#[test] +fn matching_cidr_bypasses_proxy() { + assert_routing(&["127.0.0.0/8"], true); +} + +#[test] +fn wildcard_bypasses_proxy() { + assert_routing(&["*"], true); +} + +// The proxy decision must be re-made on every redirect hop, not frozen at the +// first URL. Two distinct loopback IPs give the two hops distinct host strings +// so a single `--no-proxy` entry can exclude one but not the other. + +/// Case A — start on an excluded host (direct), which redirects onto a host +/// that is *not* excluded. The post-redirect hop must go through the proxy. +/// With the decision frozen at hop 1 it stayed direct, leaking the redirected +/// request straight out instead of through the proxy. +#[test] +fn redirect_onto_proxied_host_uses_proxy() { + let proxy_hits = Arc::new(AtomicUsize::new(0)); + // Reached only if the post-redirect hop wrongly stays direct. + let target_port = spawn_server("127.0.0.1", "TARGET_DIRECT", None); + // Excluded first host (127.0.0.2) redirects to the non-excluded host. + let redirect_port = spawn_redirect_server( + "127.0.0.2", + &format!("http://127.0.0.1:{target_port}/next"), + None, + ); + let proxy_port = spawn_server("127.0.0.1", "VIA_PROXY", Some(proxy_hits.clone())); + + let url = format!("http://127.0.0.2:{redirect_port}/start"); + let proxy = format!("http://127.0.0.1:{proxy_port}"); + let out = run_cli(&url, &proxy, &["127.0.0.2"], true); + + assert!( + out.contains("VIA_PROXY"), + "post-redirect hop should have been proxied, got: {out}" + ); + assert!( + !out.contains("TARGET_DIRECT"), + "post-redirect hop must not connect directly, got: {out}" + ); + assert_eq!( + proxy_hits.load(Ordering::SeqCst), + 1, + "the proxied (second) hop should have hit the proxy exactly once" + ); +} + +/// Case B — start on a proxied host whose response redirects onto an excluded +/// host. The post-redirect hop must connect directly. With the decision frozen +/// at hop 1 it kept using the proxy, which here re-serves the redirect and +/// loops until the redirect limit instead of reaching the excluded host. +#[test] +fn redirect_onto_no_proxy_host_connects_direct() { + let proxy_hits = Arc::new(AtomicUsize::new(0)); + let direct_hits = Arc::new(AtomicUsize::new(0)); + // Excluded redirect target — reached only via a direct connection. + let target_port = spawn_server("127.0.0.2", "TARGET_DIRECT", Some(direct_hits.clone())); + // The proxy redirects every request to the excluded host. + let proxy_port = spawn_redirect_server( + "127.0.0.1", + &format!("http://127.0.0.2:{target_port}/next"), + Some(proxy_hits.clone()), + ); + + // Start host (127.0.0.1) is not excluded -> hop 1 is proxied. + let url = format!("http://127.0.0.1:{proxy_port}/start"); + let proxy = format!("http://127.0.0.1:{proxy_port}"); + let out = run_cli(&url, &proxy, &["127.0.0.2"], true); + + assert!( + out.contains("TARGET_DIRECT"), + "post-redirect hop should have reached the excluded host directly, got: {out}" + ); + assert_eq!( + direct_hits.load(Ordering::SeqCst), + 1, + "the excluded host should have been connected to directly exactly once" + ); + assert_eq!( + proxy_hits.load(Ordering::SeqCst), + 1, + "the proxy should only have served the first hop, not the redirect loop" + ); +} + +#[test] +fn no_proxy_without_proxy_errors() { + // --no-proxy is meaningless without -x; the CLI should reject it up front + // rather than silently ignore it. Validation runs before any connection, so + // the unreachable URL is never dialed. + let output = Command::new(env!("CARGO_BIN_EXE_blasthttp")) + .arg("http://127.0.0.1:1/") + .arg("--no-proxy") + .arg("127.0.0.1") + .output() + .expect("failed to run blasthttp binary"); + + assert!( + !output.status.success(), + "expected non-zero exit when --no-proxy is used without --proxy" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("no_proxy") && stderr.contains("no proxy is configured"), + "expected a no_proxy-without-proxy error, got: {stderr}" + ); +} diff --git a/tests/python/test_no_proxy.py b/tests/python/test_no_proxy.py new file mode 100644 index 0000000..9c616cc --- /dev/null +++ b/tests/python/test_no_proxy.py @@ -0,0 +1,122 @@ +"""Pytest-asyncio tests: the proxy / no_proxy decision is re-made on every +redirect hop, not frozen at the first URL. + +These exercise the PyO3 bindings with the request(proxy=, no_proxy=, +follow_redirects=True) call shape. Two loopback IPs (127.0.0.1 / 127.0.0.2) +give the two redirect hops distinct host strings that a single no_proxy entry +can tell apart. +""" + +import asyncio + +import blasthttp +import pytest +import pytest_asyncio + + +def _http_200(body): + return (f"HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n{body}").encode() + + +def _http_302(location): + return (f"HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").encode() + + +@pytest_asyncio.fixture +async def http_factory(): + """Factory that spawns fixed-response HTTP/1.1 servers and tears them all + down on cleanup. Each call returns (port, counter), where counter is a + one-element list bumped once per accepted connection — how we tell whether a + given host was actually connected to.""" + servers = [] + + async def spawn(bind_host, response): + counter = [0] + + async def handle(reader, writer): + try: + # Drain the request head before replying. + buf = b"" + while b"\r\n\r\n" not in buf and len(buf) < 16384: + chunk = await reader.read(512) + if not chunk: + break + buf += chunk + counter[0] += 1 + writer.write(response) + await writer.drain() + finally: + writer.close() + try: + await writer.wait_closed() + except Exception: + pass + + server = await asyncio.start_server(handle, bind_host, 0) + servers.append(server) + return server.sockets[0].getsockname()[1], counter + + try: + yield spawn + finally: + for server in servers: + server.close() + for server in servers: + try: + await server.wait_closed() + except Exception: + pass + + +@pytest.fixture +def client(): + return blasthttp.BlastHTTP() + + +async def test_redirect_onto_proxied_host_reevaluates_no_proxy(client, http_factory): + """Case A (no leak): an excluded first host (direct) redirects onto a + non-excluded host. The post-redirect hop must go through the proxy, not + connect directly.""" + # Reached only if the post-redirect hop wrongly stays direct. + target_port, _ = await http_factory("127.0.0.1", _http_200("TARGET_DIRECT")) + redirect_port, _ = await http_factory("127.0.0.2", _http_302(f"http://127.0.0.1:{target_port}/next")) + proxy_port, proxy_hits = await http_factory("127.0.0.1", _http_200("VIA_PROXY")) + + r = await client.request( + f"http://127.0.0.2:{redirect_port}/start", + proxy=f"http://127.0.0.1:{proxy_port}", + no_proxy=["127.0.0.2"], + follow_redirects=True, + ) + + assert r.body == "VIA_PROXY", f"post-redirect hop should be proxied, got {r.body!r}" + assert proxy_hits[0] == 1, f"proxy should be hit once, got {proxy_hits[0]}" + + +async def test_redirect_onto_no_proxy_host_reevaluates_to_direct(client, http_factory): + """Case B: a proxied first host redirects onto an excluded host. The + post-redirect hop must connect directly, not keep using the proxy.""" + # Excluded redirect target — reached only via a direct connection. + target_port, direct_hits = await http_factory("127.0.0.2", _http_200("TARGET_DIRECT")) + # The proxy redirects every request onto the excluded host. + proxy_port, proxy_hits = await http_factory("127.0.0.1", _http_302(f"http://127.0.0.2:{target_port}/next")) + + # Start host (127.0.0.1) is not excluded -> hop 1 is proxied. + r = await client.request( + f"http://127.0.0.1:{proxy_port}/start", + proxy=f"http://127.0.0.1:{proxy_port}", + no_proxy=["127.0.0.2"], + follow_redirects=True, + ) + + assert r.body == "TARGET_DIRECT", f"post-redirect hop should be direct, got {r.body!r}" + assert direct_hits[0] == 1, f"excluded host should be hit directly once, got {direct_hits[0]}" + assert proxy_hits[0] == 1, f"proxy should only serve hop 1, got {proxy_hits[0]}" + + +async def test_no_proxy_without_proxy_raises(client): + """no_proxy without a proxy is a mistake — it should raise, not silently do + nothing. Validation runs before any connection, so the URL is never dialed.""" + with pytest.raises(RuntimeError) as exc: + await client.request("http://127.0.0.1:1/", no_proxy=["127.0.0.1"]) + assert "no_proxy" in str(exc.value)