feat(config): Separate NXDOMAIN DNS cache

CHANGELOG.md

@@ -6,6 +6,10 @@
 
 - To prevent false positives, non-public email addresses (e.g. `user@localhost`) are no longer scrubbed by default. ([#5737](https://github.com/getsentry/relay/pull/5737))
 
+**Features**:
+
+- Separate NXDOMAIN DNS cache. ([#5750](https://github.com/getsentry/relay/pull/5750))
+
 **Internal**:
 
 - Calculate and track accepted bytes per individual trace metric item via `TraceMetricByte` data category. ([#5744](https://github.com/getsentry/relay/pull/5744))

Cargo.lock

@@ -4776,6 +4776,7 @@ dependencies = [
  "flate2",
  "futures",
  "hashbrown 0.15.4",
+ "hickory-resolver",
  "http",
  "http-body-util",
  "hyper",

Cargo.toml

@@ -120,6 +120,7 @@ globset = "0.4.16"
 hash32 = "1.0.0"
 hashbrown = "0.15.4"
 hex = "0.4.3"
+hickory-resolver = "0.25.2"
 hmac = "0.12.1"
 hostname = "0.4.1"
 http = "1.3.1"

relay-config/src/config.rs

@@ -887,6 +887,10 @@ pub struct Http {
     /// the resolved entries. This helps to limit the amount of requests made to the upstream DNS
     /// server (important for K8s infrastructure).
     pub dns_cache: bool,
+    /// When `dns_cache` is enabled, controls whether NXDOMAIN responses are cached.
+    /// Set to `false` to always re-query for failed lookups.
+    /// Only has an effect when `dns_cache` is `true`.
+    pub dns_cache_nxdomain: bool,
 }
 
 impl Default for Http {
@@ -904,6 +908,7 @@ impl Default for Http {
             global_metrics: false,
             forward: true,
             dns_cache: true,
+            dns_cache_nxdomain: true,
         }
     }
 }
@@ -2225,6 +2230,12 @@ impl Config {
         self.values.http.dns_cache
     }
 
+    /// Returns `true` if relay should cache negative DNS responses (NXDOMAIN).
+    /// Only meaningful when `http_dns_cache()` returns `true`.
+    pub fn http_dns_cache_nxdomain(&self) -> bool {
+        self.values.http.dns_cache_nxdomain
+    }
+
     /// Returns the expiry timeout for cached projects.
     pub fn project_cache_expiry(&self) -> Duration {
         Duration::from_secs(self.values.cache.project_expiry.into())

relay-server/Cargo.toml

@@ -48,6 +48,7 @@ either = { workspace = true }
 flate2 = { workspace = true }
 futures = { workspace = true, features = ["async-await"] }
 hashbrown = { workspace = true }
+hickory-resolver = { workspace = true }
 http = { workspace = true }
 http-body-util = { workspace = true }
 hyper = { workspace = true }

relay-server/src/services/upstream.rs

@@ -11,10 +11,15 @@ use std::fmt;
 use std::future::Future;
 use std::pin::Pin;
 use std::sync::Arc;
+use std::net::SocketAddr;
 use std::time::Duration;
 
 use axum::response::IntoResponse;
 use bytes::Bytes;
+use hickory_resolver::config::LookupIpStrategy;
+use hickory_resolver::system_conf::read_system_conf;
+use hickory_resolver::TokioResolver;
+use hickory_resolver::name_server::TokioConnectionProvider;
 use itertools::Itertools;
 use relay_auth::{
     RegisterChallenge, RegisterRequest, RegisterResponse, Registration, SecretKey, Signature,
@@ -29,6 +34,7 @@ use relay_system::{
     AsyncResponse, FromMessage, Interface, MessageResponse, NoResponse, Sender, Service,
 };
 pub use reqwest::Method;
+use reqwest::dns::{Addrs, Name, Resolve, Resolving};
 use reqwest::header;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
@@ -841,6 +847,24 @@ impl UpstreamQuery for RegisterResponse {
     }
 }
 
+/// A custom DNS resolver backed by hickory, allowing fine-grained control over
+/// resolver options (e.g. NXDOMAIN TTL) that reqwest's `.hickory_dns()` does not expose.
+struct HickoryResolver(TokioResolver);
+
+impl Resolve for HickoryResolver {
+    fn resolve(&self, name: Name) -> Resolving {
+        let resolver = self.0.clone(); // cheap, TokioResolver is Arc-backed
+        Box::pin(async move {
+            let addrs = resolver
+                .lookup_ip(name.as_str())
+                .await?
+                .into_iter()
+                .map(|ip| SocketAddr::new(ip, 0));
+            Ok(Box::new(addrs) as Addrs)
+        })
+    }
+}
+
 /// A shared, asynchronous client to build and execute requests.
 ///
 /// The main way to send a request through this client is [`send`](Self::send).
@@ -856,16 +880,33 @@ struct SharedClient {
 impl SharedClient {
     /// Creates a new `SharedClient` instance.
     pub fn build(config: Arc<Config>) -> Self {
-        let reqwest = reqwest::ClientBuilder::new()
+        let mut builder = reqwest::ClientBuilder::new()
             .connect_timeout(config.http_connection_timeout())
             .timeout(config.http_timeout())
             // In the forward endpoint, this means that content negotiation is done twice, and the
             // response body is first decompressed by the client, then re-compressed by the server.
-            .gzip(true)
-            .hickory_dns(config.http_dns_cache())
-            .build()
-            .unwrap();
+            .gzip(true);
+
+        if config.http_dns_cache() {
+            let (system_config, mut system_opts) = read_system_conf().unwrap_or_default();
+            // Match reqwest's built-in hickory behaviour which uses Ipv4AndIpv6 (parallel,
+            // "happy eyeballs") instead of the hickory default Ipv4thenIpv6 (sequential).
+            system_opts.ip_strategy = LookupIpStrategy::Ipv4AndIpv6;
+            if !config.http_dns_cache_nxdomain() {
+                system_opts.negative_max_ttl = Some(Duration::ZERO);
+            }
+            let resolver = TokioResolver::builder_with_config(system_config, TokioConnectionProvider::default())
+                .with_options(system_opts)
+                .build();
+            builder = builder.dns_resolver(Arc::new(HickoryResolver(resolver)));
+        } else {
+            // Explicitly disable hickory so reqwest falls back to the system resolver.
+            // Without this, reqwest defaults to hickory when the feature is compiled in,
+            // ignoring the user's opt-out.
+            builder = builder.hickory_dns(false);
+        }
 
+        let reqwest = builder.build().unwrap();
         Self { config, reqwest }
     }