From 84eda4fed6410b9469c7c4bc926c3773eae78553 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 30 Jun 2026 03:03:29 -0700 Subject: [PATCH] fix: use async AMD SNP KDS fetch --- Cargo.lock | 3 + Cargo.toml | 1 + dstack-attest/src/attestation.rs | 78 ++++++-- sev-snp-qvl/Cargo.toml | 4 +- sev-snp-qvl/src/lib.rs | 296 +++++++++++++++++++++---------- verifier/Cargo.toml | 1 + verifier/src/verification.rs | 27 ++- 7 files changed, 303 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3014d376..4e3503c96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2802,6 +2802,7 @@ dependencies = [ "cc-eventlog", "clap", "dcap-qvl", + "dstack-attest", "dstack-mr", "dstack-types", "ez-hash", @@ -7467,8 +7468,10 @@ version = "0.5.11" dependencies = [ "anyhow", "hex", + "moka", "reqwest", "sev", + "tokio", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b873a6805..0d30b504d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,6 +127,7 @@ futures = "0.3.31" git-version = "0.3.9" libc = "0.2.171" log = "0.4.26" +moka = { version = "0.12.15", default-features = false, features = ["sync"] } notify = "8.0.0" rand = "0.8.5" tracing = "0.1.40" diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index 4f177343b..5c325c7fd 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -31,7 +31,7 @@ use tpm_qvl::verify::VerifiedReport as TpmVerifiedReport; // Re-export TpmQuote from tpm-types pub use tpm_types::TpmQuote; -use crate::amd_sev_snp::VerifiedAmdSnpReport; +use crate::amd_sev_snp::{AmdKdsClient, VerifiedAmdSnpReport}; pub use crate::v1::{Attestation as AttestationV1, PlatformEvidence, StackEvidence}; pub const SNP_REPORT_DATA_RANGE: std::ops::Range = 0x50..0x90; @@ -757,7 +757,27 @@ impl AttestationV1 { pub async fn verify_with_time( self, pccs_url: Option<&str>, - _now: Option, + now: Option, + ) -> Result { + self.verify_with_time_with_amd_kds_client(pccs_url, now, None) + .await + } + + /// Verify the quote with a caller-owned AMD KDS client. + pub async fn verify_with_amd_kds_client( + self, + pccs_url: Option<&str>, + amd_kds_client: &AmdKdsClient, + ) -> Result { + self.verify_with_time_with_amd_kds_client(pccs_url, None, Some(amd_kds_client)) + .await + } + + async fn verify_with_time_with_amd_kds_client( + self, + pccs_url: Option<&str>, + now: Option, + amd_kds_client: Option<&AmdKdsClient>, ) -> Result { let AttestationV1 { version: _, @@ -836,7 +856,7 @@ impl AttestationV1 { &nsm.nsm_quote, nsm_qvl::AWS_NITRO_ENCLAVES_ROOT_G1, None, - _now, + now, ) .context("NSM attestation verification failed")?; let Some(user_data) = verified_report.user_data.clone() else { @@ -862,11 +882,17 @@ impl AttestationV1 { cert_chain, mr_config, } => { - let verified = crate::amd_sev_snp::verify_amd_snp_evidence_with_kds_fallback( - report, - cert_chain, - &report_data, - )?; + let owned_kds_client; + let kds_client = match amd_kds_client { + Some(client) => client, + None => { + owned_kds_client = AmdKdsClient::new()?; + &owned_kds_client + } + }; + let verified = kds_client + .verify_evidence_with_kds_fallback(report, cert_chain, &report_data) + .await?; verify_snp_mr_config_host_data(mr_config, &verified.host_data)?; DstackVerifiedReport::DstackAmdSevSnp(verified) } @@ -1751,6 +1777,26 @@ impl Attestation { self, pccs_url: Option<&str>, now: Option, + ) -> Result { + self.verify_with_time_with_amd_kds_client(pccs_url, now, None) + .await + } + + /// Verify the quote with a caller-owned AMD KDS client. + pub async fn verify_with_amd_kds_client( + self, + pccs_url: Option<&str>, + amd_kds_client: &AmdKdsClient, + ) -> Result { + self.verify_with_time_with_amd_kds_client(pccs_url, None, Some(amd_kds_client)) + .await + } + + async fn verify_with_time_with_amd_kds_client( + self, + pccs_url: Option<&str>, + now: Option, + amd_kds_client: Option<&AmdKdsClient>, ) -> Result { let report = match &self.quote { AttestationQuote::DstackTdx(q) => { @@ -1758,11 +1804,17 @@ impl Attestation { DstackVerifiedReport::DstackTdx(report) } AttestationQuote::DstackAmdSevSnp(q) => { - let verified = crate::amd_sev_snp::verify_amd_snp_evidence_with_kds_fallback( - &q.report, - &q.cert_chain, - &self.report_data, - )?; + let owned_kds_client; + let kds_client = match amd_kds_client { + Some(client) => client, + None => { + owned_kds_client = AmdKdsClient::new()?; + &owned_kds_client + } + }; + let verified = kds_client + .verify_evidence_with_kds_fallback(&q.report, &q.cert_chain, &self.report_data) + .await?; verify_snp_mr_config_host_data(&q.mr_config, &verified.host_data)?; DstackVerifiedReport::DstackAmdSevSnp(verified) } diff --git a/sev-snp-qvl/Cargo.toml b/sev-snp-qvl/Cargo.toml index c5c152693..58e4166d7 100644 --- a/sev-snp-qvl/Cargo.toml +++ b/sev-snp-qvl/Cargo.toml @@ -13,5 +13,7 @@ description = "AMD SEV-SNP Quote Verification Library" [dependencies] anyhow.workspace = true hex.workspace = true -reqwest = { workspace = true, features = ["blocking"] } +moka.workspace = true +reqwest.workspace = true sev.workspace = true +tokio = { workspace = true, features = ["rt", "time"] } diff --git a/sev-snp-qvl/src/lib.rs b/sev-snp-qvl/src/lib.rs index 897575a8c..75add0834 100644 --- a/sev-snp-qvl/src/lib.rs +++ b/sev-snp-qvl/src/lib.rs @@ -10,7 +10,10 @@ //! must still bind the verified measurement to app/config identity before //! production key release. -use anyhow::{bail, Context, Result}; +use std::{thread, time::Duration}; + +use anyhow::{anyhow, bail, Context, Result}; +use moka::sync::Cache; use sev::certs::snp::{builtin, ca, Certificate, Chain, Verifiable}; use sev::firmware::{guest::AttestationReport, host::TcbVersion}; @@ -26,8 +29,12 @@ const VLEK_CERT_GUID: [u8; 16] = [ const CERT_TABLE_ENTRY_SIZE: usize = 24; const AMD_KDS_BASE_URL_ENV: &str = "DSTACK_AMD_KDS_BASE_URL"; const AMD_KDS_DEFAULT_BASE_URL: &str = "https://kdsintf.amd.com/vcek/v1"; +const AMD_KDS_CONNECT_TIMEOUT: Duration = Duration::from_secs(30); +const AMD_KDS_REQUEST_TIMEOUT: Duration = Duration::from_secs(120); +const AMD_KDS_CA_CACHE_CAPACITY: u64 = 16; +const AMD_KDS_VCEK_CACHE_CAPACITY: u64 = 1024; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum AmdSnpProduct { Milan, Genoa, @@ -58,7 +65,7 @@ impl AmdSnpProduct { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash)] pub struct AmdSnpTcbVersion { pub fmc: Option, pub bootloader: u8, @@ -152,6 +159,173 @@ struct AmdKdsCollateral { vcek: CertBytes, } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct AmdKdsCaCacheKey { + product: AmdSnpProduct, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct AmdKdsVcekCacheKey { + product: AmdSnpProduct, + chip_id: [u8; 64], + tcb: AmdSnpTcbVersion, +} + +#[derive(Clone)] +pub struct AmdKdsClient { + base_url: String, + http_client: reqwest::Client, + ca_cache: Cache, + vcek_cache: Cache, +} + +impl AmdKdsClient { + pub fn new() -> Result { + Self::with_base_url(amd_kds_base_url()) + } + + pub fn with_base_url(base_url: impl AsRef) -> Result { + let base_url = normalize_amd_kds_base_url(base_url.as_ref())?; + let http_client = reqwest::Client::builder() + .connect_timeout(AMD_KDS_CONNECT_TIMEOUT) + .timeout(AMD_KDS_REQUEST_TIMEOUT) + .build() + .context("failed to create amd sev-snp KDS HTTP client")?; + Ok(Self { + base_url, + http_client, + ca_cache: Cache::new(AMD_KDS_CA_CACHE_CAPACITY), + vcek_cache: Cache::new(AMD_KDS_VCEK_CACHE_CAPACITY), + }) + } + + pub async fn verify_evidence_with_kds_fallback( + &self, + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], + ) -> Result { + if !cert_chain.is_empty() { + return verify_amd_snp_evidence(report, cert_chain, expected_report_data); + } + if report.len() != 1184 { + bail!( + "invalid amd sev-snp report length: expected 1184 bytes, got {}", + report.len() + ); + } + let report_obj = AttestationReport::from_bytes(report) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + let collateral = self + .fetch_collateral_for_report(&report_obj) + .await + .context("failed to fetch amd sev-snp KDS collateral for empty cert_chain")?; + let verified = verify_amd_snp_attestation_with_cert_chain( + report, + collateral.ark, + collateral.ask, + collateral.vcek, + )?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) + } + + async fn fetch_collateral_for_report( + &self, + report: &AttestationReport, + ) -> Result { + let mut errors = Vec::new(); + for product in amd_snp_product_candidates_for_report(report)? { + match self.fetch_collateral_for_product(product, report).await { + Ok(collateral) => return Ok(collateral), + Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), + } + } + bail!( + "amd sev-snp KDS collateral unavailable for supported products: {}", + errors.join("; ") + ) + } + + async fn fetch_collateral_for_product( + &self, + product: AmdSnpProduct, + report: &AttestationReport, + ) -> Result { + let (ark, ask) = self.fetch_ca_chain(product).await?; + let mut chip_id = [0u8; 64]; + chip_id.copy_from_slice( + report + .chip_id + .as_ref() + .get(..64) + .context("amd sev-snp chip_id too short")?, + ); + let vcek = self + .fetch_vcek(product, &chip_id, report.reported_tcb.into()) + .await?; + Ok(AmdKdsCollateral { ark, ask, vcek }) + } + + async fn fetch_ca_chain(&self, product: AmdSnpProduct) -> Result<(CertBytes, CertBytes)> { + let key = AmdKdsCaCacheKey { product }; + if let Some(cached) = self.ca_cache.get(&key) { + return Ok(cached); + } + + let url = join_amd_kds_url( + &self.base_url, + &format!("{}/cert_chain", product.kds_name()), + ); + let chain = self.fetch_url(&url, "cert_chain").await?; + let (_fetched_ark, ask) = extract_ark_ask_from_amd_kds_cert_chain(&chain)?; + let collateral = (product.builtin_ark(), ask); + self.ca_cache.insert(key, collateral.clone()); + Ok(collateral) + } + + async fn fetch_vcek( + &self, + product: AmdSnpProduct, + chip_id: &[u8; 64], + tcb: AmdSnpTcbVersion, + ) -> Result { + let key = AmdKdsVcekCacheKey { + product, + chip_id: *chip_id, + tcb, + }; + if let Some(cached) = self.vcek_cache.get(&key) { + return Ok(cached); + } + + let vcek_url = amd_kds_vcek_url_with_base(&self.base_url, product, chip_id, tcb)?; + let vcek = CertBytes { + bytes: self.fetch_url(&vcek_url, "vcek").await?, + encoding: CertEncoding::Der, + }; + self.vcek_cache.insert(key, vcek.clone()); + Ok(vcek) + } + + async fn fetch_url(&self, url: &str, label: &str) -> Result> { + Ok(self + .http_client + .get(url) + .send() + .await + .with_context(|| format!("failed to request amd sev-snp {label} from {url}"))? + .error_for_status() + .with_context(|| format!("amd sev-snp {label} request failed for {url}"))? + .bytes() + .await + .with_context(|| format!("failed to read amd sev-snp {label} response"))? + .to_vec()) + } +} + pub fn verify_amd_snp_attestation( input: &AmdSnpAttestationInput<'_>, ) -> Result { @@ -321,87 +495,33 @@ pub fn verify_amd_snp_evidence_with_kds_fallback( if !cert_chain.is_empty() { return verify_amd_snp_evidence(report, cert_chain, expected_report_data); } - if report.len() != 1184 { - bail!( - "invalid amd sev-snp report length: expected 1184 bytes, got {}", - report.len() - ); - } - let report_obj = AttestationReport::from_bytes(report) - .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; - let collateral = fetch_amd_kds_collateral_for_report(&report_obj) - .context("failed to fetch amd sev-snp KDS collateral for empty cert_chain")?; - let verified = verify_amd_snp_attestation_with_cert_chain( - report, - collateral.ark, - collateral.ask, - collateral.vcek, - )?; - if &verified.report_data != expected_report_data { - bail!("amd sev-snp report_data mismatch"); - } - Ok(verified) -} - -fn fetch_amd_kds_collateral_for_report(report: &AttestationReport) -> Result { - let mut errors = Vec::new(); - for product in amd_snp_product_candidates_for_report(report)? { - match fetch_amd_kds_collateral_for_product(product, report) { - Ok(collateral) => return Ok(collateral), - Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), - } - } - bail!( - "amd sev-snp KDS collateral unavailable for supported products: {}", - errors.join("; ") - ) -} -fn fetch_amd_kds_collateral_for_product( - product: AmdSnpProduct, - report: &AttestationReport, -) -> Result { - let (ark, ask) = fetch_amd_kds_ca_chain(product)?; - let mut chip_id = [0u8; 64]; - chip_id.copy_from_slice( - report - .chip_id - .as_ref() - .get(..64) - .context("amd sev-snp chip_id too short")?, - ); - let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into())?; - let vcek = reqwest::blocking::Client::new() - .get(&vcek_url) - .send() - .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_url}"))? - .error_for_status() - .with_context(|| format!("amd sev-snp vcek request failed for {vcek_url}"))? - .bytes() - .context("failed to read amd sev-snp vcek response")? - .to_vec(); - Ok(AmdKdsCollateral { - ark, - ask, - vcek: CertBytes { - bytes: vcek, - encoding: CertEncoding::Der, - }, + let report = report.to_vec(); + let expected_report_data = *expected_report_data; + thread::spawn(move || -> Result { + let kds_client = AmdKdsClient::new()?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("failed to create amd sev-snp KDS fallback runtime")?; + runtime.block_on(kds_client.verify_evidence_with_kds_fallback( + &report, + &[], + &expected_report_data, + )) }) + .join() + .map_err(|_| anyhow!("amd sev-snp KDS fallback worker panicked"))? } -fn fetch_amd_kds_ca_chain(product: AmdSnpProduct) -> Result<(CertBytes, CertBytes)> { - let url = amd_kds_endpoint(&format!("{}/cert_chain", product.kds_name())); - let chain = reqwest::blocking::Client::new() - .get(&url) - .send() - .with_context(|| format!("failed to request amd sev-snp cert_chain from {url}"))? - .error_for_status() - .with_context(|| format!("amd sev-snp cert_chain request failed for {url}"))? - .bytes() - .context("failed to read amd sev-snp cert_chain response")?; - let (_fetched_ark, ask) = extract_ark_ask_from_amd_kds_cert_chain(&chain)?; - Ok((product.builtin_ark(), ask)) +pub async fn verify_amd_snp_evidence_with_kds_fallback_async( + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], +) -> Result { + AmdKdsClient::new()? + .verify_evidence_with_kds_fallback(report, cert_chain, expected_report_data) + .await } fn amd_kds_base_url() -> String { @@ -412,8 +532,12 @@ fn amd_kds_base_url() -> String { .unwrap_or_else(|| AMD_KDS_DEFAULT_BASE_URL.to_string()) } -fn amd_kds_endpoint(path: &str) -> String { - join_amd_kds_url(&amd_kds_base_url(), path) +fn normalize_amd_kds_base_url(base_url: &str) -> Result { + let base_url = base_url.trim().trim_end_matches('/').to_string(); + if base_url.is_empty() { + bail!("amd sev-snp KDS base URL is empty"); + } + Ok(base_url) } fn join_amd_kds_url(base_url: &str, path: &str) -> String { @@ -470,14 +594,6 @@ fn amd_snp_product_from_report(report: &AttestationReport) -> Result Result { - amd_kds_vcek_url_with_base(&amd_kds_base_url(), product, chip_id, tcb) -} - fn amd_kds_vcek_url_with_base( base_url: &str, product: AmdSnpProduct, diff --git a/verifier/Cargo.toml b/verifier/Cargo.toml index 1f0513ef4..eb4b66e59 100644 --- a/verifier/Cargo.toml +++ b/verifier/Cargo.toml @@ -36,6 +36,7 @@ tempfile.workspace = true # Internal dependencies ra-tls.workspace = true +dstack-attest.workspace = true dstack-types.workspace = true dstack-mr.workspace = true diff --git a/verifier/src/verification.rs b/verifier/src/verification.rs index 49326d30c..a2ab747d6 100644 --- a/verifier/src/verification.rs +++ b/verifier/src/verification.rs @@ -5,17 +5,19 @@ use std::{ ffi::OsStr, path::{Path, PathBuf}, + sync::OnceLock, time::Duration, }; use anyhow::{anyhow, bail, Context, Result}; use cc_eventlog::TdxEvent; +use dstack_attest::amd_sev_snp::AmdKdsClient; use dstack_mr::{RtmrLog, TdxMeasurementDetails, TdxMeasurements}; use dstack_types::VmConfig; use hex_literal::hex; use ra_tls::attestation::{ - Attestation, AttestationQuote, DstackVerifiedReport, NitroPcrs, TpmQuote, VerifiedAttestation, - VersionedAttestation, + Attestation, AttestationQuote, DstackVerifiedReport, NitroPcrs, PlatformEvidence, TpmQuote, + VerifiedAttestation, VersionedAttestation, }; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; @@ -162,6 +164,7 @@ pub struct CvmVerifier { pub download_url: String, pub download_timeout: Duration, pub pccs_url: Option, + amd_kds_client: OnceLock>, } impl CvmVerifier { @@ -176,6 +179,17 @@ impl CvmVerifier { download_url, download_timeout, pccs_url, + amd_kds_client: OnceLock::new(), + } + } + + fn amd_kds_client(&self) -> Result<&AmdKdsClient> { + match self + .amd_kds_client + .get_or_init(|| AmdKdsClient::new().map_err(|err| format!("{err:#}"))) + { + Ok(client) => Ok(client), + Err(err) => bail!("failed to create amd sev-snp KDS client: {err}"), } } @@ -437,7 +451,14 @@ impl CvmVerifier { let mut details = VerificationDetails::default(); let debug = request.debug.unwrap_or(false); - let verified = attestation.into_v1().verify(self.pccs_url.as_deref()).await; + let attestation = attestation.into_v1(); + let verified = if matches!(&attestation.platform, PlatformEvidence::SevSnp { .. }) { + attestation + .verify_with_amd_kds_client(self.pccs_url.as_deref(), self.amd_kds_client()?) + .await + } else { + attestation.verify(self.pccs_url.as_deref()).await + }; let verified_attestation = match verified { Ok(att) => { details.quote_verified = true;