Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
258 changes: 175 additions & 83 deletions dstack/Cargo.lock

Large diffs are not rendered by default.

9 changes: 5 additions & 4 deletions dstack/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -183,9 +183,10 @@ hyper-rustls = { version = "0.27", default-features = false, features = [
] }
hyperlocal = "0.9.1"
ipnet = { version = "2.11.0", features = ["serde"] }
reqwest = { version = "0.12.14", default-features = false, features = [
reqwest = { version = "0.13.4", default-features = false, features = [
"json",
"rustls-tls",
"query",
"rustls",
"charset",
"hickory-dns",
] }
Expand All @@ -202,7 +203,7 @@ url = "2.5"
# Cryptography/Security
aes-gcm = "0.10.3"
curve25519-dalek = "4.1.3"
dcap-qvl = "0.3.10"
dcap-qvl = "0.5.2"
dcap-qvl-webpki = "0.103.4"
elliptic-curve = { version = "0.13.8", features = ["pkcs8"] }
getrandom = "0.3.1"
Expand Down Expand Up @@ -231,7 +232,7 @@ alloy = { version = "1.0.32", default-features = false }
ez-hash = "1.1.0"

# Certificate/DNS
hickory-resolver = "0.24.4"
hickory-resolver = "0.26.1"
instant-acme = "0.7.2"
pem = "3.0"
rcgen = { version = "0.13.2", features = ["pem"] }
Expand Down
28 changes: 15 additions & 13 deletions dstack/certbot/src/acme_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@

use anyhow::{bail, Context, Result};
use fs_err as fs;
use hickory_resolver::error::ResolveErrorKind;
use hickory_resolver::proto::rr::RData;
use hickory_resolver::TokioResolver;
use instant_acme::{
Account, AccountCredentials, AuthorizationStatus, ChallengeType, Identifier, NewAccount,
NewOrder, Order, OrderStatus, Problem,
Expand Down Expand Up @@ -372,8 +373,6 @@ impl AcmeClient {
let start_time = std::time::Instant::now();

'outer: loop {
use hickory_resolver::AsyncResolver;

sleep(delay).await;

let elapsed = start_time.elapsed();
Expand All @@ -385,25 +384,28 @@ impl AcmeClient {
break;
}

let dns_resolver =
AsyncResolver::tokio_from_system_conf().context("failed to create dns resolver")?;
let dns_resolver = TokioResolver::builder_tokio()
.context("failed to create dns resolver")?
.build()
.context("failed to build dns resolver")?;

while let Some(challenge) = unsettled_challenges.pop() {
let expected_txt = &challenge.dns_value;
let settled = match dns_resolver.txt_lookup(&challenge.acme_domain).await {
Ok(record) => record.iter().any(|txt| {
Ok(record) => record.answers().iter().any(|answer| {
let RData::TXT(txt) = &answer.data else {
return false;
};
let actual_txt = txt.to_string();
debug!("Expected challenge: {expected_txt}, actual: {actual_txt}");
actual_txt == *expected_txt
}),
Err(err) if err.is_no_records_found() => false,
Err(err) => {
let ResolveErrorKind::NoRecordsFound { .. } = err.kind() else {
bail!(
"failed to lookup dns record {}: {err}",
challenge.acme_domain
);
};
false
bail!(
"failed to lookup dns record {}: {err}",
challenge.acme_domain
);
}
};
if !settled {
Expand Down
2 changes: 1 addition & 1 deletion dstack/ct_monitor/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ clap = { workspace = true, features = ["derive", "env"] }
hex = { workspace = true, features = ["alloc", "std"] }
hex_fmt.workspace = true
regex.workspace = true
reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls", "charset", "hickory-dns"] }
reqwest = { workspace = true, default-features = false, features = ["json", "rustls", "charset", "hickory-dns"] }
serde = { workspace = true, features = ["derive"] }
serde-human-bytes.workspace = true
serde_json.workspace = true
Expand Down
39 changes: 16 additions & 23 deletions dstack/dstack-attest/src/attestation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ use std::{borrow::Cow, time::SystemTime};
use anyhow::{anyhow, bail, Context, Result};
use cc_eventlog::{RuntimeEvent, TdxEvent};
use dcap_qvl::{
collateral::CollateralClient,
quote::{EnclaveReport, Quote, Report, TDReport10, TDReport15},
verify::VerifiedReport as TdxVerifiedReport,
};
Expand Down Expand Up @@ -44,6 +45,13 @@ const DSTACK_AMD_SEV_SNP: &str = "dstack-amd-sev-snp";
const DSTACK_GCP_TDX: &str = "dstack-gcp-tdx";
const DSTACK_NITRO_ENCLAVE: &str = "dstack-nitro-enclave";

fn dcap_collateral_client(pccs_url: Option<&str>) -> Result<CollateralClient> {
match pccs_url.map(str::trim).filter(|url| !url.is_empty()) {
Some(pccs_url) => CollateralClient::with_default_http(pccs_url),
None => CollateralClient::from_env(),
}
}

/// Path to sys-config.json in the host-shared dir.
///
/// Honors `DSTACK_HOST_SHARED_DIR` (exported by `dstack-util setup` because the
Expand Down Expand Up @@ -1419,17 +1427,10 @@ async fn verify_tdx_quote_with_events(
runtime_events: &[RuntimeEvent],
report_data: &[u8; 64],
) -> Result<TdxVerifiedReport> {
let mut pccs_url = Cow::Borrowed(pccs_url.unwrap_or_default());
if pccs_url.is_empty() {
pccs_url = match std::env::var("PCCS_URL") {
Ok(url) => Cow::Owned(url),
Err(_) => Cow::Borrowed(""),
};
}
let tdx_report =
dcap_qvl::collateral::get_collateral_and_verify(quote, Some(pccs_url.as_ref()))
.await
.context("Failed to get collateral")?;
let tdx_report = dcap_collateral_client(pccs_url)?
.fetch_and_verify(quote)
.await
.context("Failed to get collateral")?;
validate_tcb(&tdx_report)?;

let td_report = tdx_report.report.as_td10().context("no td report")?;
Expand Down Expand Up @@ -1997,18 +1998,10 @@ impl Attestation {
}

async fn verify_tdx(&self, pccs_url: Option<&str>, quote: &[u8]) -> Result<TdxVerifiedReport> {
let mut pccs_url = Cow::Borrowed(pccs_url.unwrap_or_default());
if pccs_url.is_empty() {
// try to read from PCCS_URL env var
pccs_url = match std::env::var("PCCS_URL") {
Ok(url) => Cow::Owned(url),
Err(_) => Cow::Borrowed(""),
};
}
let tdx_report =
dcap_qvl::collateral::get_collateral_and_verify(quote, Some(pccs_url.as_ref()))
.await
.context("Failed to get collateral")?;
let tdx_report = dcap_collateral_client(pccs_url)?
.fetch_and_verify(quote)
.await
.context("Failed to get collateral")?;
validate_tcb(&tdx_report)?;

let td_report = tdx_report.report.as_td10().context("no td report")?;
Expand Down
17 changes: 11 additions & 6 deletions dstack/dstack-util/src/host_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

use crate::utils::{deserialize_json_file, sha256, SysConfig};
use anyhow::{anyhow, bail, Context, Result};
use dcap_qvl::collateral::{CollateralClient, PHALA_PCCS_URL};
use dstack_types::{
shared_filenames::{HOST_SHARED_DIR, SYS_CONFIG},
Platform,
Expand Down Expand Up @@ -94,12 +95,16 @@ impl HostApi {
.map_err(|err| anyhow!("Failed to get sealing key: {err:?}"))?;

// verify the key provider quote
let verified_report = dcap_qvl::collateral::get_collateral_and_verify(
&provision.provider_quote,
self.pccs_url.as_deref(),
)
.await
.context("Failed to get quote collateral")?;
let pccs_url = self
.pccs_url
.as_deref()
.map(str::trim)
.filter(|url| !url.is_empty())
.unwrap_or(PHALA_PCCS_URL);
let verified_report = CollateralClient::with_default_http(pccs_url)?
.fetch_and_verify(&provision.provider_quote)
.await
.context("Failed to get quote collateral")?;
validate_tcb(&verified_report)?;
let sgx_report = verified_report
.report
Expand Down
41 changes: 23 additions & 18 deletions dstack/gateway/src/proxy/tls_passthough.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ use std::sync::atomic::Ordering;
use std::time::Duration;

use anyhow::{bail, Context, Result};
use hickory_resolver::{lookup::TxtLookup, TokioAsyncResolver};
use hickory_resolver::lookup::Lookup;
use hickory_resolver::proto::rr::RData;
use hickory_resolver::TokioResolver;
use proxy_protocol::ProxyHeader;
use tokio::{io::AsyncWriteExt, net::TcpStream, task::JoinSet, time::timeout};
use tracing::{debug, info, warn};
Expand Down Expand Up @@ -53,7 +55,7 @@ impl AppAddress {
pub(crate) struct AppAddressResolver {
prefix: String,
compat: bool,
resolver: TokioAsyncResolver,
resolver: TokioResolver,
}

impl AppAddressResolver {
Expand All @@ -70,36 +72,39 @@ impl AppAddressResolver {
}
}

fn app_address_tokio_resolver_from_system_conf() -> Result<TokioAsyncResolver> {
let (config, mut options) = hickory_resolver::system_conf::read_system_conf()
.context("failed to read system dns config")?;
fn app_address_tokio_resolver_from_system_conf() -> Result<TokioResolver> {
let mut builder = TokioResolver::builder_tokio().context("failed to read system dns config")?;

// App-address records may appear shortly after a CVM/app is registered.
// Reusing one resolver enables positive TXT caching, but we do not want a
// transient NXDOMAIN/NODATA response to hide a newly-added app for too
// long. Keep positive caching TTL-aware and cap negative caching.
options.cache_size = APP_ADDRESS_DNS_CACHE_SIZE;
let options = builder.options_mut();
options.cache_size = APP_ADDRESS_DNS_CACHE_SIZE as u64;
options.negative_min_ttl = Some(Duration::ZERO);
options.negative_max_ttl = Some(APP_ADDRESS_NEGATIVE_CACHE_TTL);

Ok(TokioAsyncResolver::tokio(config, options))
builder.build().context("failed to build dns resolver")
}

fn parse_lookup(lookup: &TxtLookup, sni: &str, txt_domain: &str) -> Result<Option<AppAddress>> {
let Some(txt_record) = lookup.iter().next() else {
return Ok(None);
};
let Some(data) = txt_record.txt_data().first() else {
return Ok(None);
};
AppAddress::parse(data)
.with_context(|| format!("failed to parse app address for {sni} via {txt_domain}"))
.map(Some)
fn parse_lookup(lookup: &Lookup, sni: &str, txt_domain: &str) -> Result<Option<AppAddress>> {
for answer in lookup.answers() {
let RData::TXT(txt) = &answer.data else {
continue;
};
let Some(data) = txt.txt_data.first() else {
continue;
};
return AppAddress::parse(data)
.with_context(|| format!("failed to parse app address for {sni} via {txt_domain}"))
.map(Some);
}
Ok(None)
}

/// Resolve app address by SNI. `resolver` is shared so its DNS cache is reused.
async fn resolve_app_address(
resolver: &TokioAsyncResolver,
resolver: &TokioResolver,
prefix: &str,
sni: &str,
compat: bool,
Expand Down
2 changes: 1 addition & 1 deletion dstack/nsm-qvl/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ rustls-pki-types.workspace = true
dcap-qvl-webpki = { workspace = true, features = ["alloc", "rustcrypto"] }

# CRL download
reqwest = { workspace = true, features = ["rustls-tls"] }
reqwest = { workspace = true, features = ["rustls"] }

[dev-dependencies]
nsm-attest.workspace = true
Expand Down
2 changes: 1 addition & 1 deletion dstack/ra-rpc/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ prpc.workspace = true
rocket = { workspace = true, features = ["mtls"], optional = true }
serde_json.workspace = true
tracing.workspace = true
reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "charset"], optional = true }
reqwest = { workspace = true, default-features = false, features = ["rustls", "charset"], optional = true }

ra-tls.workspace = true
bon.workspace = true
Expand Down
18 changes: 14 additions & 4 deletions dstack/ra-rpc/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ impl RaClientConfig {
.tls_sni(true)
.danger_accept_invalid_certs(self.tls_no_check)
.danger_accept_invalid_hostnames(self.tls_no_check_hostname)
.tls_built_in_root_certs(self.tls_built_in_root_certs)
.connect_timeout(Duration::from_secs(5))
.timeout(Duration::from_secs(60));
if self.cert_validator.is_some() {
Expand All @@ -60,9 +59,20 @@ impl RaClientConfig {
Identity::from_pem(identity_pem.as_bytes()).context("Failed to parse identity")?;
builder = builder.identity(identity);
}
if let Some(ca) = self.tls_ca_cert {
let ca = Certificate::from_pem(ca.as_bytes()).context("Failed to parse CA")?;
builder = builder.add_root_certificate(ca);
// reqwest 0.13 replaced tls_built_in_root_certs / add_root_certificate with
// tls_certs_merge (keep platform roots) and tls_certs_only (custom roots only).
// Hostname-check bypass also requires tls_certs_only on the rustls backend.
let ca_cert = self
.tls_ca_cert
.as_deref()
.map(|ca| Certificate::from_pem(ca.as_bytes()).context("Failed to parse CA"))
.transpose()?;
if self.tls_built_in_root_certs && !self.tls_no_check_hostname {
if let Some(ca) = ca_cert {
builder = builder.tls_certs_merge([ca]);
}
} else {
builder = builder.tls_certs_only(ca_cert);
}
Comment thread
kvinwang marked this conversation as resolved.
let client = builder.build().context("failed to create client")?;
Ok(RaClient {
Expand Down
8 changes: 6 additions & 2 deletions dstack/tdx-attest/examples/test_tdx.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

//! Test TDX attestation functions on real TDX hardware.

use dcap_qvl::collateral::get_collateral_and_verify;
use dcap_qvl::collateral::CollateralClient;
use dcap_qvl::quote::Quote;

#[tokio::main]
Expand Down Expand Up @@ -64,7 +64,11 @@ async fn main() {
" PCCS URL: {}",
pccs_url.as_deref().unwrap_or("(default)")
);
match get_collateral_and_verify(quote, pccs_url.as_deref()).await {
let verification = match CollateralClient::from_env() {
Ok(client) => client.fetch_and_verify(quote).await,
Err(err) => Err(err),
};
match verification {
Ok(verified) => {
println!(" ✓ Quote verified!");
println!(" QE status: {:?}", verified.qe_status);
Expand Down
Loading