Serve KMS ClearImageCache on a dedicated authenticated admin listener

dstack/crates/dstack-cli-core/src/config.rs

@@ -135,7 +135,6 @@ mandatory = false
 
 [core]
 cert_dir = "/kms/certs"
-admin_token_hash = ""
 # single-node: the KMS does not self-attest to its own auth API before
 # bootstrap (it still attests the genesis keys via the guest agent, and app
 # auth + per-app quote checks are unaffected).

dstack/kms/README.md

@@ -30,6 +30,41 @@ CVMs running in dstack support three boot modes:
 - `key-provider` in RTMR: `{"type": "kms", "id": "<kms-root-pubkey>"}`
 - `app-id` is equal to the address of the deployed App Smart Contract.
 
+## Admin API authentication
+
+The KMS public RPCs authenticate callers by RA-TLS attestation. Operator-facing
+admin RPCs (currently `ClearImageCache`) are instead served on a **separate admin
+listener** (`[core.admin]` in `kms.toml`) behind the same shared HTTP
+authenticator used by the VMM and gateway, configured identically to the gateway
+admin API, so the credential travels in the `Authorization: Bearer <token>` or
+`X-Admin-Token: <token>` header:
+
+```toml
+[core.admin]
+enabled = true
+address = "127.0.0.1"
+port = 8001
+# generate with: openssl rand -hex 32
+auth_token = "<token>"
+# htpasswd_file = "/etc/kms/admin.htpasswd"   # bcrypt (htpasswd -B), optional
+insecure_no_auth = false
+```
+
+The token can also be supplied via the `DSTACK_KMS_ADMIN_TOKEN` or
+`ADMIN_API_TOKEN` environment variables instead of the config file. The admin
+listener fails closed: enabled with neither `auth_token` nor `htpasswd_file`
+(and `insecure_no_auth = false`) refuses to start. Call it with, for example:
+
+```bash
+curl -X POST "http://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"image_hash": "<hash>", "config_hash": "<hash>"}'
+```
+
+This admin authentication is separate from the on-chain / webhook authorization
+below, which decides whether a CVM may boot and receive keys.
+
 ## KMS Implementation
 
 ### Components

dstack/kms/dstack-app/compose-dev.yaml

@@ -63,7 +63,6 @@ configs:
 
       [core]
       cert_dir = "/kms/certs"
-      admin_token_hash = "${ADMIN_TOKEN_HASH}"
 
       [core.image]
       verify = true

dstack/kms/dstack-app/compose-simple.yaml

@@ -39,7 +39,6 @@ configs:
 
       [core]
       cert_dir = "/kms/certs"
-      admin_token_hash = "${ADMIN_TOKEN_HASH}"
 
       [core.image]
       verify = true

dstack/kms/dstack-app/entrypoint.sh

@@ -8,9 +8,6 @@
 set -e
 
 cat <<EOF > ./kms.toml
-[core]
-admin_token_hash = "${ADMIN_TOKEN_HASH}"
-
 [core.image]
 verify = ${VERIFY_IMAGE}
 cache_dir = "./images"

dstack/kms/kms.toml

@@ -25,7 +25,6 @@ mandatory = false
 [core]
 cert_dir = "/etc/kms/certs"
 subject_postfix = ".dstack"
-admin_token_hash = ""
 site_name = ""
 # Whether trusted RPCs require the KMS to first attest itself to its own
 # auth API. Defaults to true (strict). Set to false ONLY when running KMS
@@ -53,6 +52,25 @@ cache_dir = "/usr/share/dstack/images"
 download_url = "http://localhost:8000/{OS_IMAGE_HASH}.tar.gz"
 download_timeout = "2m"
 
+# Admin API: serves operator RPCs (e.g. ClearImageCache) on a dedicated
+# listener behind the shared HTTP authenticator. Disabled by default; when
+# enabled it fails closed unless auth_token or htpasswd_file is set.
+[core.admin]
+enabled = false
+# Bind address/port for the admin listener (keep it off the public network).
+address = "127.0.0.1"
+port = 8001
+# Shared admin token required by every admin RPC. Can also be supplied via the
+# `DSTACK_KMS_ADMIN_TOKEN` or `ADMIN_API_TOKEN` env vars. Clients send it as
+# `Authorization: Bearer <token>` or the `X-Admin-Token: <token>` header.
+# Required unless insecure_no_auth = true. Generate with: openssl rand -hex 32
+auth_token = ""
+# Optional Apache bcrypt htpasswd file (create with `htpasswd -B -c file admin`).
+htpasswd_file = ""
+# Development/testing escape hatch only. Never enable this on an admin
+# interface that is reachable from the network.
+insecure_no_auth = false
+
 [core.metrics]
 # Expose unauthenticated Prometheus metrics on /metrics.
 # Disable this if the KMS RPC listener is not protected by network policy

dstack/kms/rpc/proto/kms_rpc.proto

@@ -105,14 +105,20 @@ service KMS {
   rpc GetTempCaCert(google.protobuf.Empty) returns (GetTempCaCertResponse);
   // Sign a certificate
   rpc SignCert(SignCertRequest) returns (SignCertResponse);
+}
+
+// The KMS admin RPC service. Served on a separate admin listener
+// (`[core.admin]`) behind the shared HTTP authenticator, so the credential
+// travels in the `Authorization`/`X-Admin-Token` header rather than the
+// request body.
+service Admin {
   // Clear the image cache
   rpc ClearImageCache(ClearImageCacheRequest) returns (google.protobuf.Empty);
 }
 
 message ClearImageCacheRequest {
-  string token = 1;
-  string image_hash = 2;
-  string config_hash = 3;
+  string image_hash = 1;
+  string config_hash = 2;
 }
 
 message BootstrapRequest {

dstack/kms/src/admin_auth.rs

@@ -0,0 +1,114 @@
+// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! KMS adapter for the shared API authenticator, guarding the admin listener.
+//!
+//! Unlike the KMS public RPC surface (which authenticates callers by RA-TLS
+//! attestation), the admin API is reached by operators/tooling, so it uses the
+//! same shared-token/htpasswd HTTP mechanism as the VMM and gateway. The token
+//! is configured via `core.admin.auth_token`, or the `DSTACK_KMS_ADMIN_TOKEN` /
+//! `ADMIN_API_TOKEN` environment variables.
+
+use anyhow::{bail, Result};
+use dstack_api_auth::{Authenticator, HttpAuthConfig, HttpAuthFairing};
+use rocket::Route;
+
+use crate::config::AdminConfig;
+
+const ENV_ADMIN_TOKEN: &str = "DSTACK_KMS_ADMIN_TOKEN";
+const ENV_ADMIN_TOKEN_COMPAT: &str = "ADMIN_API_TOKEN";
+
+pub struct AdminAuthFairing(HttpAuthFairing);
+
+impl AdminAuthFairing {
+    pub fn from_config(config: &AdminConfig) -> Result<Self> {
+        if config.insecure_no_auth {
+            return Ok(Self(HttpAuthFairing::new(
+                Authenticator::disabled(),
+                http_config(),
+            )));
+        }
+        let token = if !config.auth_token.is_empty() {
+            config.auth_token.trim().to_owned()
+        } else {
+            std::env::var(ENV_ADMIN_TOKEN)
+                .or_else(|_| std::env::var(ENV_ADMIN_TOKEN_COMPAT))
+                .unwrap_or_default()
+                .trim()
+                .to_owned()
+        };
+        if token.is_empty() && config.htpasswd_file.as_os_str().is_empty() {
+            bail!(
+                "admin API is enabled but neither auth_token nor htpasswd_file is configured; \
+                 set core.admin.auth_token, {ENV_ADMIN_TOKEN}, {ENV_ADMIN_TOKEN_COMPAT}, \
+                 core.admin.htpasswd_file, or insecure_no_auth = true (testing only)"
+            );
+        }
+        let mut auth = Authenticator::from_tokens([token]);
+        if !config.htpasswd_file.as_os_str().is_empty() {
+            auth = auth.with_htpasswd_file(&config.htpasswd_file)?;
+        }
+        Ok(Self(HttpAuthFairing::new(auth, http_config())))
+    }
+}
+
+fn http_config() -> HttpAuthConfig {
+    HttpAuthConfig {
+        realm: "dstack-kms admin".into(),
+        token_header: Some("X-Admin-Token".into()),
+        allow_get_query_token: false,
+    }
+}
+
+#[rocket::async_trait]
+impl rocket::fairing::Fairing for AdminAuthFairing {
+    fn info(&self) -> rocket::fairing::Info {
+        self.0.info()
+    }
+    async fn on_request(&self, req: &mut rocket::Request<'_>, data: &mut rocket::Data<'_>) {
+        self.0.on_request(req, data).await
+    }
+}
+
+pub fn routes() -> Vec<Route> {
+    dstack_api_auth::routes()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn enabled_without_credentials_fails_closed() {
+        // the token can also come from the environment; make sure neither var is
+        // set so this exercises the missing-credential path.
+        std::env::remove_var(ENV_ADMIN_TOKEN);
+        std::env::remove_var(ENV_ADMIN_TOKEN_COMPAT);
+        let cfg = AdminConfig {
+            enabled: true,
+            ..Default::default()
+        };
+        assert!(AdminAuthFairing::from_config(&cfg).is_err());
+    }
+
+    #[test]
+    fn insecure_no_auth_is_allowed() {
+        let cfg = AdminConfig {
+            enabled: true,
+            insecure_no_auth: true,
+            ..Default::default()
+        };
+        assert!(AdminAuthFairing::from_config(&cfg).is_ok());
+    }
+
+    #[test]
+    fn configured_token_builds() {
+        let cfg = AdminConfig {
+            enabled: true,
+            auth_token: "secret".into(),
+            ..Default::default()
+        };
+        assert!(AdminAuthFairing::from_config(&cfg).is_ok());
+    }
+}

dstack/kms/src/admin_service.rs

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! KMS admin RPC service, served on the `[core.admin]` listener behind the
+//! shared HTTP authenticator. Callers are authenticated by the admin token /
+//! htpasswd via `Authorization`/`X-Admin-Token`, so the handlers do no token
+//! checks of their own.
+
+use anyhow::Result;
+use dstack_kms_rpc::{
+    admin_server::{AdminRpc, AdminServer},
+    ClearImageCacheRequest,
+};
+use ra_rpc::{CallContext, RpcCall};
+
+use crate::main_service::KmsState;
+
+pub struct AdminRpcHandler {
+    state: KmsState,
+}
+
+impl AdminRpc for AdminRpcHandler {
+    async fn clear_image_cache(self, request: ClearImageCacheRequest) -> Result<()> {
+        self.state
+            .clear_image_cache(&request.image_hash, &request.config_hash)
+    }
+}
+
+impl RpcCall<KmsState> for AdminRpcHandler {
+    type PrpcService = AdminServer<Self>;
+
+    fn construct(context: CallContext<'_, KmsState>) -> Result<Self> {
+        Ok(AdminRpcHandler {
+            state: context.state.clone(),
+        })
+    }
+}

dstack/kms/src/config.rs

@@ -59,8 +59,6 @@ pub(crate) struct KmsConfig {
     /// mirroring `sev_snp_key_release`.
     #[serde(default)]
     pub aws_nitro_tpm_key_release: bool,
-    #[serde(with = "serde_human_bytes")]
-    pub admin_token_hash: Vec<u8>,
     #[serde(default)]
     pub site_name: String,
     /// Whether trusted RPCs require the KMS to first attest itself to its
@@ -70,6 +68,10 @@ pub(crate) struct KmsConfig {
     #[serde(default = "default_true")]
     pub enforce_self_authorization: bool,
     pub metrics: MetricsConfig,
+    /// Admin API listener + authentication. The admin RPCs (e.g.
+    /// `ClearImageCache`) are served here, behind the shared HTTP authenticator.
+    #[serde(default)]
+    pub admin: AdminConfig,
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -78,6 +80,29 @@ pub(crate) struct MetricsConfig {
     pub enabled: bool,
 }
 
+/// Admin API listener + authentication, mirroring the gateway `[core.admin]`
+/// section. The listen `address`/`port` are read from the same `[core.admin]`
+/// section by Rocket. The token travels in the `Authorization`/`X-Admin-Token`
+/// header.
+#[derive(Debug, Clone, Default, Deserialize)]
+pub(crate) struct AdminConfig {
+    /// Whether to serve the admin API at all.
+    #[serde(default)]
+    pub enabled: bool,
+    /// Shared admin token required to call any admin RPC. Can also be supplied
+    /// via `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN`. Required unless
+    /// `insecure_no_auth = true`.
+    #[serde(default)]
+    pub auth_token: String,
+    /// Optional Apache bcrypt htpasswd file, accepted in addition to the token.
+    #[serde(default)]
+    pub htpasswd_file: PathBuf,
+    /// Development-only escape hatch: serve the admin API with no auth. Never
+    /// enable on a network-reachable listener.
+    #[serde(default)]
+    pub insecure_no_auth: bool,
+}
+
 fn default_true() -> bool {
     true
 }
@@ -160,3 +185,23 @@ pub(crate) struct OnboardConfig {
     pub enabled: bool,
     pub auto_bootstrap_domain: String,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn default_config_parses_with_admin_disabled_and_no_hash() {
+        let figment = load_config_figment(None);
+        let config: KmsConfig = figment
+            .focus("core")
+            .extract()
+            .expect("kms.toml must parse into KmsConfig");
+        assert!(!config.admin.enabled, "admin must be off by default");
+        assert!(
+            config.admin.auth_token.is_empty(),
+            "default admin token must be empty (fail-closed)"
+        );
+        assert!(!config.admin.insecure_no_auth);
+    }
+}