Bump opendal to 0.56.0 and fix fallout (fixes local GCS cache usage)

Cargo.toml

@@ -80,19 +80,23 @@ memmap2 = "0.9.4"
 mime = "0.3"
 number_prefix = "0.4"
 object = "0.37"
-opendal = { version = "0.55.0", optional = true, default-features = false }
+opendal = { version = "0.57.0", optional = true, default-features = false }
+opendal-layer-logging = { version = "0.57.0", optional = true }
+# Not published on crates.io yet, so it is pulled directly from git at the same
+# revision as the patched `opendal`/`opendal-layer-logging` below. Provides
+# `ReqwestTransport`, the replacement for the removed `HttpClientLayer`.
+opendal-http-transport-reqwest = { git = "https://github.com/apache/opendal.git", rev = "931baacfbb73d50e16961ff8d97c0b688c8d054d", optional = true, default-features = false }
 openssl = { version = "0.10.75", optional = true }
 rand = "0.8.4"
 regex = "1.10.3"
-reqsign = { version = "0.18.0", optional = true }
-reqwest = { version = "0.12", features = [
+reqwest = { version = "0.13", features = [
   "json",
   "blocking",
   "stream",
-  "rustls-tls",
-  "rustls-tls-native-roots",
-  "trust-dns",
+  "rustls",
+  "hickory-dns",
 ], optional = true }
+rustls-native-certs = { version = "0.8", optional = true }
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = ">=1.0, <1.0.147" # zmij (used in 1.0.147+) requires Rust 1.84+
@@ -136,6 +140,10 @@ rouille = { version = "3.6", optional = true, default-features = false, features
 syslog = { version = "7", optional = true }
 version-compare = { version = "0.1.1", optional = true }
 
+[patch.crates-io]
+opendal = { git = "https://github.com/apache/opendal.git", rev = "931baacfbb73d50e16961ff8d97c0b688c8d054d" }
+opendal-layer-logging = { git = "https://github.com/apache/opendal.git", rev = "931baacfbb73d50e16961ff8d97c0b688c8d054d" }
+
 [dev-dependencies]
 assert_cmd = "2.0.13"
 cc = "1.0"
@@ -179,17 +187,17 @@ all = [
   "oss",
   "cos",
 ]
-azure = ["opendal/services-azblob", "reqsign", "reqwest"]
-cos = ["opendal/services-cos", "reqsign", "reqwest"]
+azure = ["opendal/services-azblob", "opendal-layer-logging", "opendal-http-transport-reqwest", "reqwest"]
+cos = ["opendal/services-cos", "opendal-layer-logging", "opendal-http-transport-reqwest", "reqwest"]
 default = ["all"]
-gcs = ["opendal/services-gcs", "reqsign", "url", "reqwest"]
-gha = ["opendal/services-ghac", "reqwest"]
-memcached = ["opendal/services-memcached"]
+gcs = ["opendal/services-gcs", "opendal-layer-logging", "opendal-http-transport-reqwest", "url", "reqwest"]
+gha = ["opendal/services-ghac", "opendal-layer-logging", "opendal-http-transport-reqwest", "reqwest"]
+memcached = ["opendal/services-memcached", "opendal-layer-logging"]
 native-zlib = []
-oss = ["opendal/services-oss", "reqsign", "reqwest"]
-redis = ["url", "opendal/services-redis"]
-s3 = ["opendal/services-s3", "reqsign", "reqwest"]
-webdav = ["opendal/services-webdav", "reqwest"]
+oss = ["opendal/services-oss", "opendal-layer-logging", "opendal-http-transport-reqwest", "reqwest"]
+redis = ["url", "opendal/services-redis", "opendal-layer-logging"]
+s3 = ["opendal/services-s3", "opendal-layer-logging", "opendal-http-transport-reqwest", "reqwest"]
+webdav = ["opendal/services-webdav", "opendal-layer-logging", "opendal-http-transport-reqwest", "reqwest"]
 # Enable features that will build a vendored version of openssl and
 # statically linked with it, instead of linking against the system-wide openssl
 # dynamically or statically.
@@ -205,6 +213,7 @@ dist-client = [
   "reqwest",
   "url",
   "sha2",
+  "rustls-native-certs",
 ]
 # Enables the sccache-dist binary
 dist-server = [
@@ -217,6 +226,7 @@ dist-server = [
   "rouille",
   "syslog",
   "version-compare",
+  "rustls-native-certs",
 ]
 # Enables dist tests with external requirements
 dist-tests = ["dist-client", "dist-server"]

src/cache/azure.rs

@@ -15,8 +15,9 @@
 
 use opendal::Operator;
 
-use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::OperationContext;
 use opendal::services::Azblob;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -31,9 +32,8 @@ impl AzureBlobCache {
             .root(key_prefix);
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/cos.rs

@@ -11,8 +11,9 @@
 // limitations under the License.
 
 use opendal::Operator;
-use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::OperationContext;
 use opendal::services::Cos;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -30,9 +31,8 @@ impl COSCache {
         }
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/gcs.rs

@@ -16,10 +16,8 @@
 use crate::cache::CacheMode;
 use crate::errors::*;
 use opendal::Operator;
-use opendal::{
-    layers::{HttpClientLayer, LoggingLayer},
-    services::Gcs,
-};
+use opendal::{services::Gcs, OperationContext};
+use opendal_layer_logging::LoggingLayer;
 use reqwest::Client;
 use serde::Deserialize;
 use url::Url;
@@ -74,9 +72,8 @@ impl GCSCache {
         }
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/gha.rs

@@ -13,8 +13,9 @@
 // limitations under the License.
 
 use opendal::Operator;
-use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::OperationContext;
 use opendal::services::Ghac;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::VERSION;
 use crate::errors::*;
@@ -41,9 +42,8 @@ impl GHACache {
         };
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/http_client.rs

@@ -1,9 +1,14 @@
-use opendal::raw::HttpClient;
+use opendal::HttpTransporter;
+use opendal_http_transport_reqwest::ReqwestTransport;
 use reqwest::ClientBuilder;
 
-/// Set the user agent (helps with monitoring on the server side)
-pub fn set_user_agent() -> HttpClient {
+/// Build an HTTP transport with a custom user agent (helps with monitoring on
+/// the server side).
+///
+/// Since opendal removed `HttpClientLayer`, a custom HTTP client is now supplied
+/// as an [`HttpTransporter`] via `OperationContext::with_http_transport`.
+pub fn set_user_agent() -> HttpTransporter {
     let user_agent = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
     let client = ClientBuilder::new().user_agent(user_agent).build().unwrap();
-    HttpClient::with(client)
+    HttpTransporter::new(ReqwestTransport::new(client))
 }

src/cache/memcached.rs

@@ -16,8 +16,8 @@
 use std::time::Duration;
 
 use opendal::Operator;
-use opendal::layers::LoggingLayer;
 use opendal::services::Memcached;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -45,9 +45,7 @@ impl MemcachedCache {
             .root(key_prefix)
             .default_ttl(Duration::from_secs(expiration.into()));
 
-        let op = Operator::new(builder)?
-            .layer(LoggingLayer::default())
-            .finish();
+        let op = Operator::new(builder)?.layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/oss.rs

@@ -11,8 +11,9 @@
 // limitations under the License.
 
 use opendal::Operator;
-use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::OperationContext;
 use opendal::services::Oss;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -37,13 +38,12 @@ impl OSSCache {
         if no_credentials {
             // Allow anonymous access to OSS so that OpenDAL will not
             // throw error when no credentials are provided.
-            builder = builder.allow_anonymous();
+            builder = builder.skip_signature();
         }
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/redis.rs

@@ -15,8 +15,8 @@
 
 use crate::errors::*;
 use opendal::Operator;
-use opendal::layers::LoggingLayer;
 use opendal::services::Redis;
+use opendal_layer_logging::LoggingLayer;
 use std::collections::HashMap;
 use std::time::Duration;
 use url::Url;
@@ -47,9 +47,7 @@ impl RedisCache {
             .map(|v| v.parse().unwrap_or_default())
             .unwrap_or_default());
 
-        let op = Operator::new(builder)?
-            .layer(LoggingLayer::default())
-            .finish();
+        let op = Operator::new(builder)?.layer(LoggingLayer::default());
         Ok(op)
     }
 
@@ -98,9 +96,7 @@ impl RedisCache {
             builder = builder.default_ttl(Duration::from_secs(ttl));
         }
 
-        let op = Operator::new(builder)?
-            .layer(LoggingLayer::default())
-            .finish();
+        let op = Operator::new(builder)?.layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/s3.rs

@@ -11,8 +11,9 @@
 // limitations under the License.
 
 use opendal::Operator;
-use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::OperationContext;
 use opendal::services::S3;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -86,7 +87,7 @@ impl S3Cache {
                 .disable_ec2_metadata()
                 // Allow anonymous access to S3 so that OpenDAL will not
                 // throw error when no credentials are provided.
-                .allow_anonymous();
+                .skip_signature();
         }
 
         if let Some(endpoint) = &self.endpoint {
@@ -98,9 +99,8 @@ impl S3Cache {
         }
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/webdav.rs

@@ -12,8 +12,9 @@
 
 use crate::errors::*;
 use opendal::Operator;
-use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::OperationContext;
 use opendal::services::Webdav;
+use opendal_layer_logging::LoggingLayer;
 
 use super::http_client::set_user_agent;
 
@@ -37,9 +38,8 @@ impl WebdavCache {
             .token(token.unwrap_or_default());
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/dist/http.rs

@@ -367,8 +367,8 @@ mod server {
 
         // Finish the certificate
         builder
-            .sign(&privkey, openssl::hash::MessageDigest::sha1())
-            .context("failed to sign x509 with sha1")?;
+            .sign(&privkey, openssl::hash::MessageDigest::sha256())
+            .context("failed to sign x509 with sha256")?;
         let cert: openssl::x509::X509 = builder.build();
         let cert_pem = cert.to_pem().context("failed to create pem from x509")?;
         let cert_digest = cert

src/util.rs

@@ -990,7 +990,14 @@ pub fn daemonize() -> Result<()> {
 /// More details could be found at https://github.com/mozilla/sccache/pull/1563
 #[cfg(any(feature = "dist-server", feature = "dist-client"))]
 pub fn new_reqwest_blocking_client() -> reqwest::blocking::Client {
-    reqwest::blocking::Client::builder()
+    let mut builder = reqwest::blocking::Client::builder();
+
+    // Query the native store; fallback if it's completely empty
+    if rustls_native_certs::load_native_certs().certs.is_empty() {
+        builder = builder.tls_certs_only(Vec::new());
+    }
+
+    builder
         .pool_max_idle_per_host(0)
         .build()
         .expect("http client must build with success")