From dd2e05d8a10e8207a723fda955ef2b864ab8e33b Mon Sep 17 00:00:00 2001 From: bplatz Date: Fri, 3 Jul 2026 13:14:54 -0400 Subject: [PATCH 1/9] feat(server,storage,nameservice): remote mounts and serving tiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serve ledgers to other Fluree instances in two tiers and let a consumer mount a remote's ledgers as read-only, locally-queryable sources. - ProxyStorage read modes: Raw fetches canonical CAS bytes via GET /storage/objects/{cid} with client-side CID verification (what makes indexed ledgers readable over the proxy — the FLKB tier has no leaf decoder on the read path); Filtered keeps the FLKB negotiation. Peer proxy mode now uses Raw. - Per-ledger serving posture: new f:servingDefaults setting group (f:serveQuery / f:serveBlocks / f:publicVisibility) in the ledger config graph, enforced on transaction-role servers only (query gate 403, blocks gate 404 on /storage/block, /storage/objects, /commits, /pack) and advertised per-caller as serving: ["query","blocks"] on NS record responses plus a coarse block in /.well-known/fluree.json. - Remote mounts: FlureeBuilder::with_remote_mount composes a CompositeNameService (prefix-routed reads with record localization, writes to mounted aliases rejected) with StorageBackend::Routed (namespace-prefix store selection at the content_store seam), so mounted ledgers get full native semantics including mixed datasets. - ProxyStorage/ProxyNameService moved to fluree-db-nameservice-sync (server re-exports keep the peer paths); from_api_base constructors for non-default API mounts; mount-prefix stripping on derived aliases. - HTTP Range on /storage/objects (206 + Content-Range, full-object CID verification before slicing) and native ranged reads in ProxyStorage. - Fix: dict-blob requests are branch-resolved server-side — @shared addresses carry only the ledger name, so non-main-branch peers previously 404'd on dict fetches; the legacy per-branch dict layout now parses too. --- fluree-db-api/src/config_resolver.rs | 60 +- fluree-db-api/src/lib.rs | 126 +++ fluree-db-core/src/ledger_config.rs | 24 + fluree-db-core/src/storage.rs | 65 +- fluree-db-nameservice-sync/src/lib.rs | 4 + .../src/proxy_nameservice.rs | 421 ++++++++++ .../src/proxy_storage.rs | 770 ++++++++++++++++++ fluree-db-nameservice/src/lib.rs | 1 + fluree-db-nameservice/src/mount.rs | 587 +++++++++++++ fluree-db-server/src/peer/mod.rs | 2 +- .../src/peer/proxy_nameservice.rs | 417 +--------- fluree-db-server/src/peer/proxy_storage.rs | 517 +----------- fluree-db-server/src/routes/admin.rs | 11 + fluree-db-server/src/routes/commits.rs | 11 + fluree-db-server/src/routes/mod.rs | 1 + fluree-db-server/src/routes/pack.rs | 10 + fluree-db-server/src/routes/query.rs | 16 +- fluree-db-server/src/routes/serving.rs | 85 ++ fluree-db-server/src/routes/storage_proxy.rs | 177 +++- fluree-db-server/src/state.rs | 8 +- fluree-db-server/tests/proxy_integration.rs | 745 ++++++++++++++++- fluree-vocab/src/lib.rs | 21 + 22 files changed, 3120 insertions(+), 959 deletions(-) create mode 100644 fluree-db-nameservice-sync/src/proxy_nameservice.rs create mode 100644 fluree-db-nameservice-sync/src/proxy_storage.rs create mode 100644 fluree-db-nameservice/src/mount.rs create mode 100644 fluree-db-server/src/routes/serving.rs diff --git a/fluree-db-api/src/config_resolver.rs b/fluree-db-api/src/config_resolver.rs index b23acd63e7..57c98e5976 100644 --- a/fluree-db-api/src/config_resolver.rs +++ b/fluree-db-api/src/config_resolver.rs @@ -26,7 +26,8 @@ use std::sync::Arc; use fluree_db_core::ledger_config::{ DatalogDefaults, FullTextDefaults, FullTextProperty, GraphConfig, GraphSourceRef, LedgerConfig, OntologyImportBinding, OverrideControl, PolicyDefaults, ReasoningDefaults, ResolvedConfig, - RollbackGuard, ShaclDefaults, TransactDefaults, TrustMode, TrustPolicy, ValidationMode, + RollbackGuard, ServingDefaults, ShaclDefaults, TransactDefaults, TrustMode, TrustPolicy, + ValidationMode, }; use fluree_db_core::{GraphDbRef, LedgerSnapshot, OverlayProvider, Sid, CONFIG_GRAPH_ID}; use fluree_db_novelty::Novelty; @@ -138,6 +139,7 @@ pub async fn resolve_ledger_config( let datalog = read_datalog_defaults(snapshot, overlay, to_t, &config_sid).await?; let transact = read_transact_defaults(snapshot, overlay, to_t, &config_sid).await?; let full_text = read_fulltext_defaults(snapshot, overlay, to_t, &config_sid).await?; + let serving = read_serving_defaults(snapshot, overlay, to_t, &config_sid).await?; let graph_overrides = read_graph_overrides(snapshot, overlay, to_t, &config_sid).await?; Ok(Some(LedgerConfig { @@ -148,6 +150,7 @@ pub async fn resolve_ledger_config( datalog, transact, full_text, + serving, graph_overrides, })) } @@ -1305,6 +1308,61 @@ async fn read_datalog_defaults( })) } +/// Read serving defaults from the LedgerConfig subject. +/// +/// Ledger-scoped group: read only off `f:LedgerConfig` (never GraphConfig) +/// and carries no override control. +async fn read_serving_defaults( + snapshot: &LedgerSnapshot, + overlay: &dyn OverlayProvider, + to_t: i64, + parent_sid: &Sid, +) -> Result> { + let group_sid = match read_ref_field( + snapshot, + overlay, + to_t, + parent_sid, + config_iris::SERVING_DEFAULTS, + ) + .await? + { + Some(sid) => sid, + None => return Ok(None), + }; + + let serve_query = read_bool_field( + snapshot, + overlay, + to_t, + &group_sid, + config_iris::SERVE_QUERY, + ) + .await?; + let serve_blocks = read_bool_field( + snapshot, + overlay, + to_t, + &group_sid, + config_iris::SERVE_BLOCKS, + ) + .await?; + let public_visibility = read_bool_field( + snapshot, + overlay, + to_t, + &group_sid, + config_iris::PUBLIC_VISIBILITY, + ) + .await?; + + Ok(Some(ServingDefaults { + serve_query, + serve_blocks, + public_visibility, + })) +} + /// Read transact defaults from a parent subject (LedgerConfig or GraphConfig). /// /// `f:transactDefaults` points to a group with `f:uniqueEnabled` (bool) and diff --git a/fluree-db-api/src/lib.rs b/fluree-db-api/src/lib.rs index c192832e73..2b3c48dcbc 100644 --- a/fluree-db-api/src/lib.rs +++ b/fluree-db-api/src/lib.rs @@ -366,6 +366,96 @@ impl std::fmt::Debug for NameServiceMode { } } +/// One remote mount for [`FlureeBuilder::with_remote_mount`]: the ledgers of +/// a remote Fluree appear locally, read-only, under `prefix/`. +/// +/// The lookup and storage are transport-agnostic — for HTTP mounts, build a +/// `ProxyNameService` and a `ProxyStorage` (raw mode, with the matching +/// local prefix) from `fluree-db-nameservice-sync` and pass them here. +#[derive(Clone)] +pub struct RemoteMountSpec { + prefix: String, + lookup: Arc, + storage: StorageBackend, +} + +impl std::fmt::Debug for RemoteMountSpec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RemoteMountSpec") + .field("prefix", &self.prefix) + .finish_non_exhaustive() + } +} + +impl RemoteMountSpec { + /// Create a mount spec for aliases under `prefix` (no trailing slash). + pub fn new( + prefix: impl Into, + lookup: Arc, + storage: impl fluree_db_core::Storage + 'static, + ) -> Self { + Self { + prefix: prefix.into(), + lookup, + storage: StorageBackend::Managed(Arc::new(storage)), + } + } + + /// The alias prefix this mount claims. + pub fn prefix(&self) -> &str { + &self.prefix + } +} + +/// Wrap a backend + nameservice pair with remote mounts. +/// +/// The nameservice becomes a [`CompositeNameService`] (prefix-routed reads, +/// local writes, mounted writes rejected) and the backend a +/// [`StorageBackend::Routed`] so mounted namespaces read through their own +/// storage. No-op when `mounts` is empty. Mounts require a read-write local +/// nameservice; on a read-only instance they are dropped with an error log. +/// +/// [`CompositeNameService`]: fluree_db_nameservice::mount::CompositeNameService +fn apply_remote_mounts( + backend: StorageBackend, + nameservice: NameServiceMode, + mounts: Vec, +) -> (StorageBackend, NameServiceMode) { + use fluree_db_core::storage::RoutedBackend; + use fluree_db_nameservice::mount::{CompositeNameService, RemoteMount}; + + if mounts.is_empty() { + return (backend, nameservice); + } + + let publisher = match nameservice { + NameServiceMode::ReadWrite(publisher) => publisher, + NameServiceMode::ReadOnly(lookup) => { + tracing::error!( + mounts = mounts.len(), + "remote mounts require a read-write local nameservice; ignoring mounts" + ); + return (backend, NameServiceMode::ReadOnly(lookup)); + } + }; + + let ns_mounts: Vec = mounts + .iter() + .map(|m| RemoteMount::new(m.prefix.clone(), Arc::clone(&m.lookup))) + .collect(); + let composite = CompositeNameService::new(publisher, ns_mounts) + .expect("mount prefixes deduplicated by FlureeBuilder::with_remote_mount"); + + let storage_mounts: Vec<(String, StorageBackend)> = + mounts.into_iter().map(|m| (m.prefix, m.storage)).collect(); + let routed = RoutedBackend::new(backend, storage_mounts); + + ( + StorageBackend::Routed(Arc::new(routed)), + NameServiceMode::ReadWrite(Arc::new(composite)), + ) +} + impl NameServiceMode { /// Get read-only nameservice access (always available). pub fn reader(&self) -> &dyn NameServiceLookup { @@ -1143,6 +1233,8 @@ pub struct FlureeBuilder { novelty_thresholds: Option, /// Remote Fluree connection registry for SERVICE federation. remote_connections: remote_service::RemoteConnectionRegistry, + /// Read-only remote mounts applied at build time (alias-prefixed). + remote_mounts: Vec, } /// Configuration for background indexing in `FlureeBuilder`. @@ -1359,6 +1451,7 @@ impl FlureeBuilder { indexing_config: Some(default_indexing_builder_config()), novelty_thresholds: None, remote_connections: remote_service::RemoteConnectionRegistry::new(), + remote_mounts: Vec::new(), } } @@ -1373,6 +1466,7 @@ impl FlureeBuilder { indexing_config: None, novelty_thresholds: None, remote_connections: remote_service::RemoteConnectionRegistry::new(), + remote_mounts: Vec::new(), } } @@ -1439,6 +1533,7 @@ impl FlureeBuilder { indexing_config: Some(default_indexing_builder_config()), novelty_thresholds: None, remote_connections: remote_service::RemoteConnectionRegistry::new(), + remote_mounts: Vec::new(), } } @@ -1631,6 +1726,7 @@ impl FlureeBuilder { indexing_config, novelty_thresholds: None, remote_connections: remote_service::RemoteConnectionRegistry::new(), + remote_mounts: Vec::new(), }) } @@ -1781,6 +1877,22 @@ impl FlureeBuilder { self } + /// Mount a remote Fluree's ledgers read-only under the spec's alias + /// prefix (e.g. prefix `acme` exposes remote `inventory:main` as + /// `acme/inventory:main`). + /// + /// Reads (nameservice lookups and CAS content) route to the mount; + /// writes to mounted aliases fail with a "read-only remote mount" error. + /// Registering a second mount with the same prefix replaces the first. + /// + /// Mounts require a read-write local nameservice; on a read-only + /// (proxy-peer) instance they are ignored with an error log. + pub fn with_remote_mount(mut self, spec: RemoteMountSpec) -> Self { + self.remote_mounts.retain(|m| m.prefix != spec.prefix); + self.remote_mounts.push(spec); + self + } + /// Build a file-backed Fluree instance /// /// Returns an error if storage_path is not set. @@ -1820,6 +1932,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, )) } @@ -1849,6 +1962,7 @@ impl FlureeBuilder { attachment_provider_cell: Self::new_attachment_provider_cell(), }, self.remote_connections, + self.remote_mounts, ) } @@ -1955,6 +2069,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, )) } @@ -1991,6 +2106,7 @@ impl FlureeBuilder { attachment_provider_cell: Self::new_attachment_provider_cell(), }, self.remote_connections, + self.remote_mounts, ) } @@ -2024,6 +2140,7 @@ impl FlureeBuilder { attachment_provider_cell: Self::new_attachment_provider_cell(), }, self.remote_connections, + self.remote_mounts, ) } @@ -2077,6 +2194,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, ) } @@ -2154,6 +2272,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, )) } @@ -2255,6 +2374,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, )) } @@ -2340,6 +2460,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, )) } @@ -2463,6 +2584,7 @@ impl FlureeBuilder { config: ConnectionConfig, parts: RuntimeParts, remote_connections: remote_service::RemoteConnectionRegistry, + remote_mounts: Vec, ) -> Fluree { let RuntimeParts { backend, @@ -2472,6 +2594,7 @@ impl FlureeBuilder { index_config, attachment_provider_cell, } = parts; + let (backend, nameservice) = apply_remote_mounts(backend, nameservice, remote_mounts); let leaflet_cache = make_leaflet_cache(&config); let governance_cache = std::sync::Arc::new(cross_ledger::GovernanceCache::new()); @@ -2618,6 +2741,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, )) } @@ -2688,6 +2812,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, )) } } @@ -2748,6 +2873,7 @@ impl FlureeBuilder { attachment_provider_cell, }, self.remote_connections, + self.remote_mounts, )) } diff --git a/fluree-db-core/src/ledger_config.rs b/fluree-db-core/src/ledger_config.rs index 38d3f22575..e0b804c65a 100644 --- a/fluree-db-core/src/ledger_config.rs +++ b/fluree-db-core/src/ledger_config.rs @@ -37,6 +37,9 @@ pub struct LedgerConfig { pub transact: Option, /// Full-text indexing defaults (`f:fullTextDefaults`). pub full_text: Option, + /// Serving-posture defaults (`f:servingDefaults`). Ledger-scoped: + /// never merged per-graph and not subject to override control. + pub serving: Option, /// Per-graph config overrides (`f:graphOverrides`). pub graph_overrides: Vec, } @@ -148,6 +151,27 @@ pub struct DatalogDefaults { pub override_control: OverrideControl, } +/// Serving-posture defaults from the config graph (`f:servingDefaults`). +/// +/// Declares which serving tiers the ledger's origin server offers to callers. +/// Gates apply only on the origin (transaction-role) serving surface: a +/// read-only peer or mount that holds the ledger's blocks always queries its +/// own copy freely. `None` fields mean "allowed" (an unconfigured ledger is +/// fully served); `public_visibility` defaults to false (token required). +/// +/// Ledger-scoped: lives only on `f:LedgerConfig`, ignored on `f:GraphConfig`, +/// and not subject to override control — it changes only by transacting the +/// config graph. +#[derive(Debug, Clone, Default)] +pub struct ServingDefaults { + /// `f:serveQuery` — origin executes queries for this ledger. + pub serve_query: Option, + /// `f:serveBlocks` — origin serves raw CAS blocks (storage proxy). + pub serve_blocks: Option, + /// `f:publicVisibility` — ledger is discoverable/readable without a token. + pub public_visibility: Option, +} + /// Full-text indexing defaults from the config graph (`f:fullTextDefaults`). /// /// Declares properties whose string values should be BM25-indexed without diff --git a/fluree-db-core/src/storage.rs b/fluree-db-core/src/storage.rs index 3fffc0b40b..4d96c11773 100644 --- a/fluree-db-core/src/storage.rs +++ b/fluree-db-core/src/storage.rs @@ -699,6 +699,55 @@ pub enum StorageBackend { Managed(Arc), /// Append-only content-addressed storage (IPFS). Permanent(Arc), + /// Namespace-routed composition: mounted alias prefixes read through + /// their own backend, everything else uses the default backend. + Routed(Arc), +} + +/// Namespace-prefix routing table for [`StorageBackend::Routed`]. +/// +/// Each mount claims a ledger-name prefix (e.g. `"acme"` routes +/// `acme/inventory:main` and every namespace under `acme/`). Store selection +/// happens in [`StorageBackend::content_store`], the single point where a +/// ledger's namespace is bound to a store — so ledger loading, branched-store +/// ancestry walks, and default-context reads all route without changes. +/// +/// Admin operations (delete, list) apply only to the default backend; mounts +/// are read-only remote content. +pub struct RoutedBackend { + default: StorageBackend, + mounts: Vec<(String, StorageBackend)>, +} + +impl RoutedBackend { + /// Compose a default backend with `(prefix, backend)` mounts. + pub fn new(default: StorageBackend, mounts: Vec<(String, StorageBackend)>) -> Self { + Self { default, mounts } + } + + /// Select the backend owning `namespace_id` (a ledger ID or name). + fn backend_for(&self, namespace_id: &str) -> &StorageBackend { + self.mounts + .iter() + .find(|(prefix, _)| { + namespace_id + .strip_prefix(prefix.as_str()) + .is_some_and(|rest| rest.starts_with('/')) + }) + .map_or(&self.default, |(_, backend)| backend) + } +} + +impl Debug for RoutedBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RoutedBackend") + .field("default", &self.default) + .field( + "mounts", + &self.mounts.iter().map(|(p, _)| p).collect::>(), + ) + .finish() + } } impl StorageBackend { @@ -707,33 +756,41 @@ impl StorageBackend { /// /// For `Managed` backends, this constructs a [`StorageContentStore`] that /// maps CIDs to physical addresses under the namespace. For `Permanent` - /// backends, the inner store is returned directly. + /// backends, the inner store is returned directly. For `Routed` backends, + /// the namespace's owning backend (mount or default) is selected first. pub fn content_store(&self, namespace_id: &str) -> Arc { match self { StorageBackend::Managed(storage) => { Arc::new(content_store_for(storage.clone(), namespace_id)) } StorageBackend::Permanent(store) => Arc::clone(store), + StorageBackend::Routed(routed) => { + routed.backend_for(namespace_id).content_store(namespace_id) + } } } /// Get the underlying raw storage for admin operations (delete, list). /// - /// Returns `Some` for `Managed` backends, `None` for `Permanent`. + /// Returns `Some` for `Managed` backends (and the default backend of + /// `Routed`), `None` for `Permanent`. pub fn admin_storage(&self) -> Option<&dyn Storage> { match self { StorageBackend::Managed(storage) => Some(storage.as_ref()), StorageBackend::Permanent(_) => None, + StorageBackend::Routed(routed) => routed.default.admin_storage(), } } /// Clone the admin storage as an owned `Arc`, if available. /// - /// Returns `Some` for `Managed` backends, `None` for `Permanent`. + /// Returns `Some` for `Managed` backends (and the default backend of + /// `Routed`), `None` for `Permanent`. pub fn admin_storage_cloned(&self) -> Option> { match self { StorageBackend::Managed(storage) => Some(Arc::clone(storage)), StorageBackend::Permanent(_) => None, + StorageBackend::Routed(routed) => routed.default.admin_storage_cloned(), } } } @@ -743,6 +800,7 @@ impl Debug for StorageBackend { match self { StorageBackend::Managed(s) => f.debug_tuple("Managed").field(s).finish(), StorageBackend::Permanent(s) => f.debug_tuple("Permanent").field(s).finish(), + StorageBackend::Routed(s) => f.debug_tuple("Routed").field(s).finish(), } } } @@ -752,6 +810,7 @@ impl Clone for StorageBackend { match self { StorageBackend::Managed(s) => StorageBackend::Managed(Arc::clone(s)), StorageBackend::Permanent(s) => StorageBackend::Permanent(Arc::clone(s)), + StorageBackend::Routed(s) => StorageBackend::Routed(Arc::clone(s)), } } } diff --git a/fluree-db-nameservice-sync/src/lib.rs b/fluree-db-nameservice-sync/src/lib.rs index e902a9d7cd..725252ab6e 100644 --- a/fluree-db-nameservice-sync/src/lib.rs +++ b/fluree-db-nameservice-sync/src/lib.rs @@ -26,6 +26,8 @@ pub mod driver; pub mod error; pub mod origin; pub mod pack_client; +pub mod proxy_nameservice; +pub mod proxy_storage; mod server_sse; pub mod watch; pub mod watch_poll; @@ -45,6 +47,8 @@ pub use pack_client::{ fetch_and_ingest_pack, ingest_pack_frame, ingest_pack_stream, ingest_pack_stream_with_header, peek_pack_header, PackIngestResult, }; +pub use proxy_nameservice::ProxyNameService; +pub use proxy_storage::{ProxyReadMode, ProxyStorage}; pub use watch::{RemoteEvent, RemoteWatch}; pub use watch_poll::PollRemoteWatch; pub use watch_sse::SseRemoteWatch; diff --git a/fluree-db-nameservice-sync/src/proxy_nameservice.rs b/fluree-db-nameservice-sync/src/proxy_nameservice.rs new file mode 100644 index 0000000000..c46cf1a309 --- /dev/null +++ b/fluree-db-nameservice-sync/src/proxy_nameservice.rs @@ -0,0 +1,421 @@ +//! Proxy nameservice implementation for peer mode +//! +//! Fetches nameservice records via the transaction server's `/v1/fluree/storage/ns/{alias}` +//! endpoint instead of direct file access. This allows peers to operate without storage +//! credentials. + +use async_trait::async_trait; +use fluree_db_nameservice::{NameServiceError, NsRecord, Result}; +use reqwest::{Client, StatusCode}; +use serde::Deserialize; +use std::fmt::Debug; +use std::time::Duration; + +/// NameService implementation that proxies lookups through the transaction server +#[derive(Clone)] +pub struct ProxyNameService { + client: Client, + api_base: String, + token: String, +} + +impl Debug for ProxyNameService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProxyNameService") + .field("api_base", &self.api_base) + .finish_non_exhaustive() + } +} + +/// Response from nameservice lookup endpoint. +/// Must match `NsRecordResponse` from routes/storage_proxy.rs. +/// +/// Uses `#[serde(default)]` on optional fields so that missing JSON keys +/// deserialize as `None` rather than failing — this keeps the peer +/// forward-compatible when the server adds new fields. +#[derive(Debug, Deserialize)] +struct NsRecordResponse { + #[serde(default)] + name: Option, + branch: String, + commit_head_id: Option, + commit_t: i64, + index_head_id: Option, + index_t: i64, + #[serde(default)] + default_context: Option, + retracted: bool, + #[serde(default)] + config_id: Option, + /// Parent branch this branch was forked from. Required for + /// peers to build the `BranchedContentStore` that resolves + /// commits inherited from the source branch — without it, all + /// reads of inherited commits 404 even when the ledger is + /// reachable. + #[serde(default)] + source_branch: Option, + /// Number of child branches forked from this one. Defaults to + /// 0 when an older server omits the field. + #[serde(default)] + branches: u32, +} + +impl NsRecordResponse { + /// Convert to NsRecord, using the original lookup key as the ledger_id. + /// + /// When the server omits `name`, derive it from `lookup_key` by splitting + /// on `:` (e.g., `"books:main"` → `"books"`). This avoids copying the full + /// `ledger_id` (which includes the branch) into the `name` field. + fn into_ns_record(self, lookup_key: &str) -> NsRecord { + use fluree_db_core::ContentId; + + let derived_name = self.name.unwrap_or_else(|| { + lookup_key + .split_once(':') + .map(|(name, _branch)| name.to_string()) + .unwrap_or_else(|| lookup_key.to_string()) + }); + + NsRecord { + // ledger_id is the key used for lookup (may differ from name) + ledger_id: lookup_key.to_string(), + name: derived_name, + branch: self.branch, + commit_head_id: self + .commit_head_id + .and_then(|s| s.parse::().ok()), + config_id: self.config_id.and_then(|s| s.parse::().ok()), + commit_t: self.commit_t, + index_head_id: self.index_head_id.and_then(|s| s.parse::().ok()), + index_t: self.index_t, + default_context: self + .default_context + .and_then(|s| s.parse::().ok()), + retracted: self.retracted, + source_branch: self.source_branch, + branches: self.branches, + } + } +} + +impl ProxyNameService { + /// Create a new proxy nameservice client + /// + /// # Arguments + /// + /// * `base_url` - Base URL of the transaction server (e.g., `https://tx.fluree.internal:8090`) + /// * `token` - Bearer token for authentication (with `fluree.storage.*` claims) + pub fn new(base_url: String, token: String) -> Self { + // Server root → default versioned API base. + let api_base = format!("{}/v1/fluree", base_url.trim_end_matches('/')); + Self::from_api_base(api_base, token) + } + + /// Create a proxy nameservice client from a full API base URL (e.g. + /// `https://data.example.com/v1/fluree`), as stored by `fluree remote + /// add` or advertised via discovery's `api_base_url`. Use this instead + /// of [`new`](Self::new) when the API may be mounted under a + /// non-default prefix. + pub fn from_api_base(api_base: String, token: String) -> Self { + let client = Client::builder() + .timeout(Duration::from_secs(30)) // 30 seconds for NS lookups + .build() + .expect("Failed to create proxy nameservice client"); + + Self { + client, + api_base: api_base.trim_end_matches('/').to_string(), + token, + } + } + + /// Build the nameservice lookup endpoint URL + fn ns_url(&self, alias: &str) -> String { + format!( + "{}/storage/ns/{}", + self.api_base, + urlencoding::encode(alias) + ) + } +} + +#[async_trait] +impl fluree_db_nameservice::RefLookup for ProxyNameService { + async fn get_ref( + &self, + _ledger_id: &str, + _kind: fluree_db_nameservice::RefKind, + ) -> Result> { + Err(NameServiceError::storage( + "get_ref not supported in proxy mode".to_string(), + )) + } +} + +#[async_trait] +impl fluree_db_nameservice::StatusLookup for ProxyNameService { + async fn get_status( + &self, + _ledger_id: &str, + ) -> Result> { + Err(NameServiceError::storage( + "get_status not supported in proxy mode".to_string(), + )) + } +} + +#[async_trait] +impl fluree_db_nameservice::ConfigLookup for ProxyNameService { + async fn get_config( + &self, + _ledger_id: &str, + ) -> Result> { + Err(NameServiceError::storage( + "get_config not supported in proxy mode".to_string(), + )) + } +} + +#[async_trait] +impl fluree_db_nameservice::NameServiceLookup for ProxyNameService { + async fn lookup(&self, ledger_id: &str) -> Result> { + let url = self.ns_url(ledger_id); + + let response = self + .client + .get(&url) + .header("Authorization", format!("Bearer {}", self.token)) + .send() + .await + .map_err(|e| { + NameServiceError::storage(format!("Nameservice proxy request failed: {e}")) + })?; + + let status = response.status(); + + match status { + StatusCode::OK => { + let ns_response: NsRecordResponse = response.json().await.map_err(|e| { + NameServiceError::storage(format!("Failed to parse NS response: {e}")) + })?; + Ok(Some(ns_response.into_ns_record(ledger_id))) + } + StatusCode::NOT_FOUND => Ok(None), + StatusCode::UNAUTHORIZED => Err(NameServiceError::storage(format!( + "Nameservice proxy authentication failed for {ledger_id}: check token validity" + ))), + StatusCode::FORBIDDEN => { + // Not in token scope - treat as not found (no existence leak) + Ok(None) + } + _ => Err(NameServiceError::storage(format!( + "Nameservice proxy unexpected status {status} for {ledger_id}" + ))), + } + } + + async fn all_records(&self) -> Result> { + // Peers use SSE for discovery, not all_records() + // Return empty - this is intentional for proxy mode + // The peer maintains its own view of known ledgers via SSE events + Ok(Vec::new()) + } +} + +// No `BranchLifecycle` impl for `ProxyNameService`: peer mode has no +// authority to mutate the nameservice. Branch lifecycle requests are +// served by the upstream transaction server (via its own HTTP routes, +// not through this trait). + +#[async_trait] +impl fluree_db_nameservice::GraphSourceLookup for ProxyNameService { + async fn lookup_graph_source( + &self, + _graph_source_id: &str, + ) -> Result> { + Ok(None) // Proxy doesn't have local graph source records + } + + async fn lookup_any(&self, resource_id: &str) -> Result { + // Delegate to the ledger lookup endpoint. Graph-source + // discovery isn't exposed through the storage proxy today, + // so a non-ledger resource still reports NotFound — but a + // real ledger record now resolves correctly instead of + // always returning NotFound and breaking every caller that + // routes through `GraphSourceLookup::lookup_any`. + use fluree_db_nameservice::{NameServiceLookup, NsLookupResult}; + match self.lookup(resource_id).await? { + Some(record) => Ok(NsLookupResult::Ledger(record)), + None => Ok(NsLookupResult::NotFound), + } + } + + async fn all_graph_source_records( + &self, + ) -> Result> { + Ok(Vec::new()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_proxy_nameservice_debug() { + let ns = ProxyNameService::new( + "http://localhost:8090".to_string(), + "test-token".to_string(), + ); + let debug = format!("{ns:?}"); + assert!(debug.contains("ProxyNameService")); + assert!(debug.contains("localhost:8090")); + // Token should NOT be in debug output + assert!(!debug.contains("test-token")); + } + + #[test] + fn test_ns_url() { + let ns = ProxyNameService::new( + "http://localhost:8090".to_string(), + "test-token".to_string(), + ); + assert_eq!( + ns.ns_url("books:main"), + "http://localhost:8090/v1/fluree/storage/ns/books%3Amain" + ); + } + + #[test] + fn test_ns_url_no_special_chars() { + let ns = ProxyNameService::new( + "http://localhost:8090".to_string(), + "test-token".to_string(), + ); + // Alias without colon doesn't need encoding + assert_eq!( + ns.ns_url("books"), + "http://localhost:8090/v1/fluree/storage/ns/books" + ); + } + + #[test] + fn test_ns_record_conversion() { + let response = NsRecordResponse { + name: Some("books".to_string()), + branch: "main".to_string(), + commit_head_id: None, + commit_t: 42, + index_head_id: None, + index_t: 40, + default_context: None, + retracted: false, + config_id: None, + source_branch: None, + branches: 0, + }; + + // Use the lookup key as ledger_id (simulating lookup("books")) + let record = response.into_ns_record("books"); + // ledger_id should be the lookup key, not the alias + assert_eq!(record.ledger_id, "books"); + assert_eq!(record.name, "books"); + assert_eq!(record.branch, "main"); + assert_eq!(record.commit_t, 42); + assert_eq!(record.index_t, 40); + assert!(!record.retracted); + // default_context is not exposed via proxy API + assert!(record.default_context.is_none()); + } + + /// Regression for finding #11: `source_branch` and `branches` + /// must round-trip through the proxy so peers can build the + /// `BranchedContentStore` for forked branches. Earlier code + /// hardcoded `source_branch: None` regardless of what the + /// server sent, breaking every read of an inherited commit. + #[test] + fn ns_record_conversion_preserves_branch_lineage() { + let response = NsRecordResponse { + name: Some("books".to_string()), + branch: "feature".to_string(), + commit_head_id: None, + commit_t: 7, + index_head_id: None, + index_t: 5, + default_context: None, + retracted: false, + config_id: None, + source_branch: Some("main".to_string()), + branches: 3, + }; + + let record = response.into_ns_record("books:feature"); + assert_eq!(record.source_branch.as_deref(), Some("main")); + assert_eq!(record.branches, 3); + } + + /// Old-server compatibility: when the wire response omits + /// `source_branch`/`branches`, deserialization defaults them + /// to `None`/`0` rather than failing. + #[test] + fn ns_record_response_deserializes_without_lineage_fields() { + let json = r#"{ + "name": "books", + "branch": "main", + "commit_head_id": null, + "commit_t": 0, + "index_head_id": null, + "index_t": 0, + "retracted": false + }"#; + let response: NsRecordResponse = + serde_json::from_str(json).expect("response without lineage fields parses"); + assert!(response.source_branch.is_none()); + assert_eq!(response.branches, 0); + } + + #[test] + fn test_ns_record_name_derived_from_lookup_key() { + // When server omits `name`, derive it by splitting lookup_key on ':' + let response = NsRecordResponse { + name: None, + branch: "main".to_string(), + commit_head_id: None, + commit_t: 10, + index_head_id: None, + index_t: 0, + default_context: None, + retracted: false, + config_id: None, + source_branch: None, + branches: 0, + }; + + let record = response.into_ns_record("books:main"); + assert_eq!(record.ledger_id, "books:main"); + // name should be "books", NOT "books:main" + assert_eq!(record.name, "books"); + } + + #[test] + fn test_ns_record_name_no_branch_in_lookup_key() { + // When lookup_key has no colon, use it as-is + let response = NsRecordResponse { + name: None, + branch: "main".to_string(), + commit_head_id: None, + commit_t: 10, + index_head_id: None, + index_t: 0, + default_context: None, + retracted: false, + config_id: None, + source_branch: None, + branches: 0, + }; + + let record = response.into_ns_record("books"); + assert_eq!(record.ledger_id, "books"); + assert_eq!(record.name, "books"); + } +} diff --git a/fluree-db-nameservice-sync/src/proxy_storage.rs b/fluree-db-nameservice-sync/src/proxy_storage.rs new file mode 100644 index 0000000000..77fbd946e4 --- /dev/null +++ b/fluree-db-nameservice-sync/src/proxy_storage.rs @@ -0,0 +1,770 @@ +//! Proxy storage implementation for peer mode +//! +//! Fetches ledger content over HTTP from another Fluree server instead of +//! direct storage access. This allows peers to operate without storage +//! credentials (no S3 access, no filesystem mount). +//! +//! Two read modes are supported, selected at construction via [`ProxyReadMode`]: +//! +//! ## `ProxyReadMode::Raw` — full-access CAS reads +//! +//! Fetches canonical CAS bytes via `GET /v1/fluree/storage/objects/{cid}` and +//! verifies every payload against its CID client-side before returning it. +//! Leaf blocks arrive as raw FLI3, so the binary index reader consumes them +//! directly. Requires a token whose scope grants **full read access** to the +//! ledger — the server serves raw index content without policy filtering on +//! this endpoint. +//! +//! ## `ProxyReadMode::Filtered` — policy-filtered reads +//! +//! Fetches through `POST /v1/fluree/storage/block`, where the server always +//! returns decoded, policy-filtered flakes (FLKB format) for leaf blocks — +//! raw FLI3 leaf bytes are never returned. Both `read_bytes()` and +//! `read_bytes_hint()` use flakes-first content negotiation: they request +//! `application/x-fluree-flakes` first, falling back to +//! `application/octet-stream` on 406 (for non-leaf blocks like commits and +//! branches). Payloads are identity-specific and NOT verifiable against their +//! CID; integrity rests on TLS + bearer auth. Note that no in-tree reader +//! currently decodes FLKB leaves — this tier is the transport for future +//! fine-grained (row-level filtered) peer access. + +use async_trait::async_trait; +use fluree_db_core::error::{Error as CoreError, Result}; +use fluree_db_core::format_ledger_id; +use fluree_db_core::storage::ReadHint; +use fluree_db_core::{ + ContentAddressedWrite, ContentId, ContentKind, ContentWriteResult, StorageRead, StorageWrite, + CODEC_FLUREE_COMMIT, CODEC_FLUREE_DICT_BLOB, CODEC_FLUREE_GARBAGE, CODEC_FLUREE_INDEX_BRANCH, + CODEC_FLUREE_INDEX_LEAF, CODEC_FLUREE_INDEX_ROOT, CODEC_FLUREE_LEDGER_CONFIG, CODEC_FLUREE_TXN, +}; +use reqwest::{Client, StatusCode}; +use serde::Serialize; +use std::fmt::Debug; +use std::time::Duration; + +/// How [`ProxyStorage`] fetches ledger content from the origin server. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProxyReadMode { + /// Raw CAS reads via `GET /storage/objects/{cid}`: canonical bytes, + /// integrity-verified against the CID client-side. Requires full read + /// access to the ledger. + Raw, + /// Policy-filtered reads via `POST /storage/block`: leaf blocks arrive + /// as FLKB (decoded, filtered flakes), non-leaf blocks as raw bytes. + /// Payloads are not CID-verifiable. + Filtered, +} + +/// Storage implementation that proxies reads through the transaction server +#[derive(Clone)] +pub struct ProxyStorage { + client: Client, + api_base: String, + token: String, + mode: ProxyReadMode, + /// Mount prefix under which the remote's ledgers appear locally + /// (e.g. `"acme"` makes remote `inventory:main` appear as + /// `acme/inventory:main`). Stripped from locally-derived aliases + /// before requests go to the remote. + local_prefix: Option, +} + +impl Debug for ProxyStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProxyStorage") + .field("api_base", &self.api_base) + .field("mode", &self.mode) + .finish_non_exhaustive() + } +} + +/// Request body for the `/storage/block` endpoint. +/// +/// Both fields are required. The `cid` and `ledger` are derived from +/// the storage address via [`cid_and_ledger_from_address()`]. +#[derive(Debug, Serialize)] +struct BlockRequest { + cid: String, + ledger: String, +} + +/// Try to derive a CID string and ledger alias from a Fluree address. +/// +/// Parses the canonical `fluree:{method}://{alias_path}/{kind_dir}/{hash}.{ext}` +/// format produced by [`fluree_db_core::content_address()`]. +/// +/// Returns `(cid, ledger_alias)` on success, or `None` if the format +/// is unrecognized (in which case the caller should fall back to address-based). +fn cid_and_ledger_from_address(address: &str) -> Option<(ContentId, String)> { + // Strip `fluree:{method}://` prefix + let rest = address.strip_prefix("fluree:")?; + let sep_pos = rest.find("://")?; + let path = &rest[sep_pos + 3..]; + + let parts: Vec<&str> = path.split('/').collect(); + let n = parts.len(); + if n < 3 { + return None; + } + + // Extract hash hex from the filename stem (before extension) + let filename = parts[n - 1]; + let (hash_hex, _ext) = filename.rsplit_once('.')?; + + // Determine the multicodec and where the alias ends based on directory + // structure. These patterns match those generated by `content_path()` in + // fluree-db-core; `branch_in_path` is false for the @shared dict layout, + // whose addresses carry only the ledger name. + let (codec, alias_end, branch_in_path) = if n >= 2 && parts[n - 2] == "commit" { + (CODEC_FLUREE_COMMIT, n - 2, true) + } else if n >= 2 && parts[n - 2] == "txn" { + (CODEC_FLUREE_TXN, n - 2, true) + } else if n >= 2 && parts[n - 2] == "config" { + (CODEC_FLUREE_LEDGER_CONFIG, n - 2, true) + } else if n >= 3 && parts[n - 3] == "index" && parts[n - 2] == "roots" { + (CODEC_FLUREE_INDEX_ROOT, n - 3, true) + } else if n >= 3 && parts[n - 3] == "index" && parts[n - 2] == "garbage" { + (CODEC_FLUREE_GARBAGE, n - 3, true) + } else if n >= 4 && parts[n - 4] == "index" && parts[n - 3] == "objects" { + match parts[n - 2] { + "branches" => (CODEC_FLUREE_INDEX_BRANCH, n - 4, true), + "leaves" => (CODEC_FLUREE_INDEX_LEAF, n - 4, true), + // Legacy per-branch dict layout (pre-@shared): + // {name}/{branch}/index/objects/dicts/{hash}.dict + "dicts" => (CODEC_FLUREE_DICT_BLOB, n - 4, true), + _ => return None, + } + } else if n >= 3 && parts[n - 2] == "dicts" && parts[n - 3] == "@shared" { + // DictBlob: {name}/@shared/dicts/{hash}.{ext} — no branch in the + // path; the default branch stands in and the server resolves it to + // a live branch of the name. + (CODEC_FLUREE_DICT_BLOB, n - 3, false) + } else { + return None; + }; + + // Reconstruct ledger ID from the alias segments before the kind directory. + let ledger_id = if branch_in_path { + if alias_end < 2 { + return None; + } + let branch = parts[alias_end - 1]; + let name = parts[..alias_end - 1].join("/"); + format_ledger_id(&name, branch) + } else { + if alias_end < 1 { + return None; + } + let name = parts[..alias_end].join("/"); + format_ledger_id(&name, fluree_db_core::DEFAULT_BRANCH) + }; + + // Build CID from codec + hex digest + let cid = ContentId::from_hex_digest(codec, hash_hex)?; + Some((cid, ledger_id)) +} + +/// Internal result type for fetch operations +/// +/// This avoids using CoreError for the 406 case, which is an internal +/// retry condition rather than a user-facing error. +enum FetchOutcome { + /// Successfully fetched bytes + Success(Vec), + /// Server returned 406 Not Acceptable (format not available) + NotAcceptable, + /// Fetch failed with an error + Error(CoreError), +} + +impl ProxyStorage { + /// Create a new proxy storage client + /// + /// # Arguments + /// + /// * `base_url` - Base URL of the transaction server (e.g., `https://tx.fluree.internal:8090`) + /// * `token` - Bearer token for authentication (with `fluree.storage.*` claims) + /// * `mode` - Read mode: [`ProxyReadMode::Raw`] for CID-verified canonical + /// bytes (full-access tokens), [`ProxyReadMode::Filtered`] for + /// policy-filtered FLKB leaf payloads + pub fn new(base_url: String, token: String, mode: ProxyReadMode) -> Self { + // Server root → default versioned API base. + let api_base = format!("{}/v1/fluree", base_url.trim_end_matches('/')); + Self::from_api_base(api_base, token, mode) + } + + /// Create a proxy storage client from a full API base URL (e.g. + /// `https://data.example.com/v1/fluree`), as stored by `fluree remote add` + /// or advertised via discovery's `api_base_url`. Use this instead of + /// [`new`](Self::new) when the API may be mounted under a non-default + /// prefix. + pub fn from_api_base(api_base: String, token: String, mode: ProxyReadMode) -> Self { + let client = Client::builder() + .timeout(Duration::from_secs(60)) // 1 minute for block reads + .build() + .expect("Failed to create proxy storage client"); + + Self { + client, + api_base: api_base.trim_end_matches('/').to_string(), + token, + mode, + local_prefix: None, + } + } + + /// Set the mount prefix under which the remote's ledgers appear locally. + /// + /// With prefix `"acme"`, a read of address + /// `fluree:proxy://acme/inventory/main/commit/x.fc` is requested from the + /// remote as ledger `inventory:main`. + #[must_use] + pub fn with_local_prefix(mut self, prefix: impl Into) -> Self { + self.local_prefix = Some(prefix.into()); + self + } + + /// Rewrite a locally-derived ledger alias to the remote's own alias by + /// stripping the mount prefix (`acme/inventory:main` → `inventory:main`). + fn remote_ledger(&self, ledger: String) -> String { + match &self.local_prefix { + Some(prefix) => ledger + .strip_prefix(prefix.as_str()) + .and_then(|rest| rest.strip_prefix('/')) + .map(str::to_string) + .unwrap_or(ledger), + None => ledger, + } + } + + /// Build the storage block endpoint URL + fn block_url(&self) -> String { + format!("{}/storage/block", self.api_base) + } + + /// Build the CAS object endpoint URL for a CID + fn object_url(&self, cid: &ContentId) -> String { + format!("{}/storage/objects/{}", self.api_base, cid) + } + + /// Fetch canonical CAS bytes via `GET /storage/objects/{cid}` and verify + /// them against the CID before returning. + async fn fetch_raw_object(&self, address: &str) -> Result> { + let (cid, ledger) = cid_and_ledger_from_address(address).ok_or_else(|| { + CoreError::storage(format!("Cannot derive CID from address: {address}")) + })?; + let ledger = self.remote_ledger(ledger); + + let response = match self + .client + .get(self.object_url(&cid)) + .query(&[("ledger", ledger.as_str())]) + .header("Authorization", format!("Bearer {}", self.token)) + .send() + .await + { + Ok(r) => r, + Err(e) => { + return Err(if e.is_timeout() { + CoreError::io(format!("Storage proxy timeout for {address}: {e}")) + } else if e.is_connect() { + CoreError::io(format!( + "Storage proxy connection failed for {address}: {e}" + )) + } else { + CoreError::io(format!("Storage proxy request failed for {address}: {e}")) + }); + } + }; + + let status = response.status(); + match status { + StatusCode::OK => { + let bytes = response.bytes().await.map_err(|e| { + CoreError::io(format!("Failed to read response body for {address}: {e}")) + })?; + if !crate::origin::verify_object_integrity(&cid, &bytes) { + return Err(CoreError::storage(format!( + "Integrity verification failed for {address} (cid {cid})" + ))); + } + Ok(bytes.to_vec()) + } + // 403 → NotFound parity with the server's no-existence-leak behavior + StatusCode::NOT_FOUND | StatusCode::FORBIDDEN => Err(CoreError::not_found(address)), + StatusCode::UNAUTHORIZED => Err(CoreError::storage(format!( + "Storage proxy authentication failed for {address}: check token validity" + ))), + s if s.is_server_error() => Err(CoreError::io(format!( + "Storage proxy server error for {address}: {status}" + ))), + _ => Err(CoreError::storage(format!( + "Storage proxy unexpected status {status} for {address}" + ))), + } + } + + /// Fetch with flakes-first content negotiation + /// + /// Tries `application/x-fluree-flakes` first. If the server returns 406 + /// (format not available for this block type), falls back to raw bytes. + async fn fetch_prefer_flakes(&self, address: &str) -> Result> { + // Try flakes format first + match self + .fetch_with_accept(address, "application/x-fluree-flakes") + .await + { + FetchOutcome::Success(bytes) => Ok(bytes), + FetchOutcome::NotAcceptable => { + // 406 = not a leaf or policy filtering not applicable + // Fall back to raw bytes + tracing::debug!( + address = %address, + "Flakes format not available, falling back to raw bytes" + ); + match self + .fetch_with_accept(address, "application/octet-stream") + .await + { + FetchOutcome::Success(bytes) => Ok(bytes), + FetchOutcome::NotAcceptable => { + // Should not happen for octet-stream, but handle anyway + Err(CoreError::storage(format!( + "Storage proxy: octet-stream also rejected for {address}" + ))) + } + FetchOutcome::Error(e) => Err(e), + } + } + FetchOutcome::Error(e) => Err(e), + } + } + + /// Internal fetch with a specific Accept header + /// + /// Returns `FetchOutcome` to distinguish between success, 406, and errors + /// without using string-based error detection. + /// + /// Derives CID+ledger from the address. Failure to parse is a hard error + /// (all storage addresses in the system use the canonical format). + async fn fetch_with_accept(&self, address: &str, accept: &str) -> FetchOutcome { + let url = self.block_url(); + let (cid, ledger) = match cid_and_ledger_from_address(address) { + Some(pair) => pair, + None => { + return FetchOutcome::Error(CoreError::storage(format!( + "Cannot derive CID from address: {address}" + ))); + } + }; + let body = BlockRequest { + cid: cid.to_string(), + ledger: self.remote_ledger(ledger), + }; + + let response = match self + .client + .post(&url) + .header("Authorization", format!("Bearer {}", self.token)) + .header("Accept", accept) + .json(&body) + .send() + .await + { + Ok(r) => r, + Err(e) => { + let err = if e.is_timeout() { + CoreError::io(format!("Storage proxy timeout for {address}: {e}")) + } else if e.is_connect() { + CoreError::io(format!( + "Storage proxy connection failed for {address}: {e}" + )) + } else { + CoreError::io(format!("Storage proxy request failed for {address}: {e}")) + }; + return FetchOutcome::Error(err); + } + }; + + let status = response.status(); + + match status { + StatusCode::OK => match response.bytes().await { + Ok(bytes) => FetchOutcome::Success(bytes.to_vec()), + Err(e) => FetchOutcome::Error(CoreError::io(format!( + "Failed to read response body for {address}: {e}" + ))), + }, + StatusCode::NOT_FOUND => FetchOutcome::Error(CoreError::not_found(address)), + StatusCode::NOT_ACCEPTABLE => { + // 406 - format not available, signal for retry with different Accept + FetchOutcome::NotAcceptable + } + StatusCode::UNAUTHORIZED => FetchOutcome::Error(CoreError::storage(format!( + "Storage proxy authentication failed for {address}: check token validity" + ))), + StatusCode::FORBIDDEN => { + // Address not in token scope - treat as not found (no existence leak) + FetchOutcome::Error(CoreError::not_found(address)) + } + s if s.is_server_error() => FetchOutcome::Error(CoreError::io(format!( + "Storage proxy server error for {address}: {status}" + ))), + _ => FetchOutcome::Error(CoreError::storage(format!( + "Storage proxy unexpected status {status} for {address}" + ))), + } + } +} + +#[async_trait] +impl StorageRead for ProxyStorage { + async fn read_bytes(&self, address: &str) -> Result> { + match self.mode { + // Raw mode: canonical CAS bytes, CID-verified client-side. + ProxyReadMode::Raw => self.fetch_raw_object(address).await, + // Filtered mode: flakes-first negotiation for deterministic + // behavior across block types: + // - Leaves → FLKB (policy-filtered flakes) + // - Non-leaves → raw bytes (via 406 fallback to octet-stream) + ProxyReadMode::Filtered => self.fetch_prefer_flakes(address).await, + } + } + + async fn read_byte_range(&self, address: &str, range: std::ops::Range) -> Result> { + if range.start >= range.end { + return Ok(Vec::new()); + } + // Filtered payloads are transport-encoded (FLKB), so byte ranges + // don't apply — fall back to full read + slice. + if self.mode != ProxyReadMode::Raw { + let full = self.read_bytes(address).await?; + let start = range.start as usize; + if start >= full.len() { + return Ok(Vec::new()); + } + let end = (range.end as usize).min(full.len()); + return Ok(full[start..end].to_vec()); + } + + let (cid, ledger) = cid_and_ledger_from_address(address).ok_or_else(|| { + CoreError::storage(format!("Cannot derive CID from address: {address}")) + })?; + let ledger = self.remote_ledger(ledger); + + let response = self + .client + .get(self.object_url(&cid)) + .query(&[("ledger", ledger.as_str())]) + .header("Authorization", format!("Bearer {}", self.token)) + // HTTP ranges are inclusive; ours are half-open. + .header("Range", format!("bytes={}-{}", range.start, range.end - 1)) + .send() + .await + .map_err(|e| { + CoreError::io(format!("Storage proxy request failed for {address}: {e}")) + })?; + + let status = response.status(); + match status { + // Partial payloads can't be CID-verified client-side; the server + // verifies the full object against the CID before slicing. + StatusCode::PARTIAL_CONTENT => { + let bytes = response.bytes().await.map_err(|e| { + CoreError::io(format!("Failed to read response body for {address}: {e}")) + })?; + Ok(bytes.to_vec()) + } + // Server ignored the Range header (older server): verify the + // full object and slice locally. + StatusCode::OK => { + let bytes = response.bytes().await.map_err(|e| { + CoreError::io(format!("Failed to read response body for {address}: {e}")) + })?; + if !crate::origin::verify_object_integrity(&cid, &bytes) { + return Err(CoreError::storage(format!( + "Integrity verification failed for {address} (cid {cid})" + ))); + } + let start = range.start as usize; + if start >= bytes.len() { + return Ok(Vec::new()); + } + let end = (range.end as usize).min(bytes.len()); + Ok(bytes[start..end].to_vec()) + } + // Range start past the object length — matches the default + // implementation's empty-slice semantics. + StatusCode::RANGE_NOT_SATISFIABLE => Ok(Vec::new()), + StatusCode::NOT_FOUND | StatusCode::FORBIDDEN => Err(CoreError::not_found(address)), + StatusCode::UNAUTHORIZED => Err(CoreError::storage(format!( + "Storage proxy authentication failed for {address}: check token validity" + ))), + s if s.is_server_error() => Err(CoreError::io(format!( + "Storage proxy server error for {address}: {status}" + ))), + _ => Err(CoreError::storage(format!( + "Storage proxy unexpected status {status} for {address}" + ))), + } + } + + fn supports_ranged_reads(&self) -> bool { + // Raw mode issues true HTTP Range requests; the filtered tier + // full-fetches and slices. + self.mode == ProxyReadMode::Raw + } + + async fn read_bytes_hint(&self, address: &str, hint: ReadHint) -> Result> { + match self.mode { + // Raw mode always returns canonical bytes; the FLKB preference + // only applies to the filtered tier. + ProxyReadMode::Raw => self.fetch_raw_object(address).await, + ProxyReadMode::Filtered => match hint { + ReadHint::AnyBytes => self.read_bytes(address).await, + ReadHint::PreferLeafFlakes => self.fetch_prefer_flakes(address).await, + // Future ReadHint variants fall back to default + _ => self.read_bytes(address).await, + }, + } + } + + async fn exists(&self, address: &str) -> Result { + // In v1, implement exists as try-read + // This is correct but slightly less efficient than a HEAD request + // The server currently only supports POST for blocks anyway + match self.read_bytes(address).await { + Ok(_) => Ok(true), + Err(CoreError::NotFound(_)) => Ok(false), + Err(e) => Err(e), + } + } + + async fn list_prefix(&self, _prefix: &str) -> Result> { + // ProxyStorage is read-only and doesn't support listing + Err(CoreError::storage( + "ProxyStorage does not support list_prefix".to_string(), + )) + } +} + +#[async_trait] +impl StorageWrite for ProxyStorage { + async fn write_bytes(&self, _address: &str, _bytes: &[u8]) -> Result<()> { + Err(CoreError::storage( + "ProxyStorage is read-only (writes must go to the transaction server)".to_string(), + )) + } + + async fn delete(&self, _address: &str) -> Result<()> { + Err(CoreError::storage( + "ProxyStorage is read-only (deletes must go to the transaction server)".to_string(), + )) + } +} + +#[async_trait] +impl ContentAddressedWrite for ProxyStorage { + async fn content_write_bytes_with_hash( + &self, + _kind: ContentKind, + _ledger_alias: &str, + _content_hash_hex: &str, + _bytes: &[u8], + ) -> Result { + Err(CoreError::storage( + "ProxyStorage is read-only (writes must go to the transaction server)".to_string(), + )) + } + + async fn content_write_bytes( + &self, + _kind: ContentKind, + _ledger_alias: &str, + _bytes: &[u8], + ) -> Result { + Err(CoreError::storage( + "ProxyStorage is read-only (writes must go to the transaction server)".to_string(), + )) + } +} + +impl fluree_db_core::StorageMethod for ProxyStorage { + fn storage_method(&self) -> &'static str { + "proxy" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use fluree_db_core::ContentKind; + + #[test] + fn test_proxy_storage_debug() { + let storage = ProxyStorage::new( + "http://localhost:8090".to_string(), + "test-token".to_string(), + ProxyReadMode::Filtered, + ); + let debug = format!("{storage:?}"); + assert!(debug.contains("ProxyStorage")); + assert!(debug.contains("localhost:8090")); + // Token should NOT be in debug output + assert!(!debug.contains("test-token")); + } + + #[test] + fn test_block_url() { + let storage = ProxyStorage::new( + "http://localhost:8090".to_string(), + "test-token".to_string(), + ProxyReadMode::Raw, + ); + assert_eq!( + storage.block_url(), + "http://localhost:8090/v1/fluree/storage/block" + ); + let id = ContentId::new(ContentKind::Commit, b"url test"); + assert_eq!( + storage.object_url(&id), + format!("http://localhost:8090/v1/fluree/storage/objects/{id}") + ); + } + + #[test] + fn test_block_url_with_trailing_slash() { + let storage = ProxyStorage::new( + "http://localhost:8090/".to_string(), + "test-token".to_string(), + ProxyReadMode::Raw, + ); + // Should work but might have double slash - that's okay for URLs + assert!(storage.block_url().contains("/v1/fluree/storage/block")); + } + + // ======================================================================== + // cid_and_ledger_from_address tests + // ======================================================================== + + /// Round-trip helper: build an address via `content_address`, then verify + /// `cid_and_ledger_from_address` recovers the correct CID and ledger. + fn assert_roundtrip(kind: ContentKind, alias: &str, data: &[u8]) { + let id = ContentId::new(kind, data); + let address = fluree_db_core::content_address("file", kind, alias, &id.digest_hex()); + let (cid, ledger) = cid_and_ledger_from_address(&address).expect("should parse address"); + assert_eq!(cid, id, "CID mismatch for {address}"); + assert_eq!(ledger, alias, "ledger mismatch for {address}"); + } + + #[test] + fn test_parse_commit_address() { + assert_roundtrip(ContentKind::Commit, "mydb:main", b"commit data"); + } + + #[test] + fn test_parse_txn_address() { + assert_roundtrip(ContentKind::Txn, "mydb:main", b"txn data"); + } + + #[test] + fn test_parse_index_root_address() { + assert_roundtrip(ContentKind::IndexRoot, "mydb:main", b"root data"); + } + + #[test] + fn test_parse_index_branch_address() { + assert_roundtrip(ContentKind::IndexBranch, "mydb:main", b"branch data"); + } + + #[test] + fn test_parse_index_leaf_address() { + assert_roundtrip(ContentKind::IndexLeaf, "mydb:main", b"leaf data"); + } + + #[test] + fn test_parse_dict_blob_address() { + use fluree_db_core::DictKind; + assert_roundtrip( + ContentKind::DictBlob { + dict: DictKind::Graphs, + }, + "mydb:main", + b"dict data", + ); + } + + #[test] + fn test_parse_config_address() { + assert_roundtrip(ContentKind::LedgerConfig, "mydb:main", b"config data"); + } + + #[test] + fn test_parse_garbage_address() { + assert_roundtrip(ContentKind::GarbageRecord, "mydb:main", b"gc data"); + } + + #[test] + fn test_parse_address_with_s3_method() { + let id = ContentId::new(ContentKind::Commit, b"s3 test"); + let address = fluree_db_core::content_address( + "s3", + ContentKind::Commit, + "prod:main", + &id.digest_hex(), + ); + let (cid, ledger) = cid_and_ledger_from_address(&address).expect("should parse s3 address"); + assert_eq!(cid, id); + assert_eq!(ledger, "prod:main"); + } + + #[test] + fn test_remote_ledger_prefix_strip() { + let storage = ProxyStorage::new( + "http://localhost:8090".to_string(), + "test-token".to_string(), + ProxyReadMode::Raw, + ) + .with_local_prefix("acme"); + assert_eq!( + storage.remote_ledger("acme/inventory:main".to_string()), + "inventory:main" + ); + // Non-matching aliases pass through unchanged. + assert_eq!( + storage.remote_ledger("other:main".to_string()), + "other:main" + ); + // A name that merely starts with the prefix string (no separator) + // is not stripped. + assert_eq!( + storage.remote_ledger("acmecorp:main".to_string()), + "acmecorp:main" + ); + // Without a prefix configured, everything passes through. + let plain = ProxyStorage::new( + "http://localhost:8090".to_string(), + "test-token".to_string(), + ProxyReadMode::Raw, + ); + assert_eq!( + plain.remote_ledger("acme/inventory:main".to_string()), + "acme/inventory:main" + ); + } + + #[test] + fn test_parse_address_not_fluree() { + assert!(cid_and_ledger_from_address("https://example.com/foo").is_none()); + } + + #[test] + fn test_parse_address_too_short() { + assert!(cid_and_ledger_from_address("fluree:file://a/b").is_none()); + } + + #[test] + fn test_parse_address_unknown_kind_dir() { + assert!(cid_and_ledger_from_address("fluree:file://mydb/main/unknown/abc.bin").is_none()); + } +} diff --git a/fluree-db-nameservice/src/lib.rs b/fluree-db-nameservice/src/lib.rs index 22ab803b8f..356ef14465 100644 --- a/fluree-db-nameservice/src/lib.rs +++ b/fluree-db-nameservice/src/lib.rs @@ -24,6 +24,7 @@ mod event_bus; pub mod file; pub mod ledger_config; pub mod memory; +pub mod mount; mod notifying; pub(crate) mod ns_format; pub mod storage_ns; diff --git a/fluree-db-nameservice/src/mount.rs b/fluree-db-nameservice/src/mount.rs new file mode 100644 index 0000000000..25df70ef6c --- /dev/null +++ b/fluree-db-nameservice/src/mount.rs @@ -0,0 +1,587 @@ +//! Remote mounts: alias-namespaced composition of nameservices. +//! +//! A [`CompositeNameService`] presents one nameservice built from a local +//! read-write nameservice plus any number of read-only remote mounts. Each +//! mount claims an alias prefix: with mount `acme`, the remote's ledger +//! `inventory:main` appears locally as `acme/inventory:main`. +//! +//! Reads route by prefix — a mounted alias is stripped of its prefix, +//! resolved against the mount's lookup (typically a `ProxyNameService` from +//! `fluree-db-nameservice-sync`), and the returned records are re-localized +//! so every downstream consumer (ledger cache, content-store namespacing, +//! branched-store ancestry walks) keys on the local alias. Writes to mounted +//! aliases are rejected with a clear error; writes to everything else +//! delegate to the local publisher. This gives per-alias write capability on +//! top of [`NameServiceMode`]'s instance-level split. +//! +//! Mount prefixes shadow local ledgers whose names start with the same +//! segment — pick mount names that don't collide with local ledger name +//! prefixes (enforced by [`CompositeNameService::new`] only against other +//! mounts, since local ledgers can be created later). + +use crate::{ + AdminPublisher, BranchLifecycle, CasResult, CommitPublisher, ConfigCasResult, ConfigLookup, + ConfigPublisher, ConfigValue, GraphSourceLookup, GraphSourcePublisher, GraphSourceRecord, + GraphSourceType, IndexPublisher, LedgerLifecycle, NameServiceError, NameServiceLookup, + NameServicePublisher, NsLookupResult, NsRecord, NsRecordSnapshot, RefKind, RefLookup, + RefPublisher, RefValue, Result, StatusCasResult, StatusLookup, StatusPublisher, StatusValue, +}; +use async_trait::async_trait; +use fluree_db_core::ContentId; +use std::fmt::Debug; +use std::sync::Arc; + +/// One read-only remote mount: an alias prefix and the lookup that serves it. +#[derive(Clone)] +pub struct RemoteMount { + prefix: String, + lookup: Arc, +} + +impl Debug for RemoteMount { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RemoteMount") + .field("prefix", &self.prefix) + .finish_non_exhaustive() + } +} + +impl RemoteMount { + /// Create a mount serving aliases under `prefix` (no trailing slash). + pub fn new(prefix: impl Into, lookup: Arc) -> Self { + Self { + prefix: prefix.into(), + lookup, + } + } + + /// The alias prefix this mount claims. + pub fn prefix(&self) -> &str { + &self.prefix + } + + /// Strip this mount's prefix from a local alias, returning the remote + /// alias. `None` if the alias is not under this mount. + fn remote_alias<'a>(&self, local: &'a str) -> Option<&'a str> { + local + .strip_prefix(self.prefix.as_str()) + .and_then(|rest| rest.strip_prefix('/')) + } + + /// Rewrite a remote record so all identity fields carry the local + /// (prefixed) alias. Branch names are unprefixed and stay as-is. + fn localize_record(&self, mut record: NsRecord) -> NsRecord { + record.ledger_id = format!("{}/{}", self.prefix, record.ledger_id); + record.name = format!("{}/{}", self.prefix, record.name); + record + } + + /// Rewrite a remote graph-source record onto the local alias namespace. + fn localize_graph_source(&self, mut record: GraphSourceRecord) -> GraphSourceRecord { + record.graph_source_id = format!("{}/{}", self.prefix, record.graph_source_id); + record.name = format!("{}/{}", self.prefix, record.name); + record + } +} + +/// A nameservice composed of a local read-write nameservice and N read-only +/// remote mounts, routed by alias prefix. +/// +/// Implements the full [`NameServicePublisher`] surface: reads route to the +/// owning mount (or the local nameservice), writes to mounted aliases fail +/// with a "read-only remote mount" error, and writes to local aliases +/// delegate to the local publisher. +#[derive(Clone)] +pub struct CompositeNameService { + local: Arc, + mounts: Vec, +} + +impl Debug for CompositeNameService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CompositeNameService") + .field("mounts", &self.mounts) + .finish_non_exhaustive() + } +} + +impl CompositeNameService { + /// Compose a local publisher with remote mounts. + /// + /// # Errors + /// Returns an error if two mounts claim the same prefix. + pub fn new(local: Arc, mounts: Vec) -> Result { + for (i, a) in mounts.iter().enumerate() { + for b in mounts.iter().skip(i + 1) { + if a.prefix == b.prefix { + return Err(NameServiceError::storage(format!( + "duplicate remote mount prefix '{}'", + a.prefix + ))); + } + } + } + Ok(Self { local, mounts }) + } + + /// The mounts this composite routes to. + pub fn mounts(&self) -> &[RemoteMount] { + &self.mounts + } + + /// Find the mount owning `alias`, with the remote-side alias. + fn mount_for<'a>(&self, alias: &'a str) -> Option<(&RemoteMount, &'a str)> { + self.mounts + .iter() + .find_map(|m| m.remote_alias(alias).map(|remote| (m, remote))) + } + + fn reject_mounted_write(&self, alias: &str) -> Option { + self.mount_for(alias).map(|(mount, _)| { + NameServiceError::storage(format!( + "'{alias}' is a read-only remote mount ('{}'); writes must go to the origin server", + mount.prefix + )) + }) + } +} + +// --------------------------------------------------------------------------- +// Read surface: route by prefix +// --------------------------------------------------------------------------- + +#[async_trait] +impl NameServiceLookup for CompositeNameService { + async fn lookup(&self, ledger_id: &str) -> Result> { + match self.mount_for(ledger_id) { + Some((mount, remote)) => Ok(mount + .lookup + .lookup(remote) + .await? + .map(|r| mount.localize_record(r))), + None => self.local.lookup(ledger_id).await, + } + } + + async fn all_records(&self) -> Result> { + // Local records plus whatever each mount can enumerate. Proxy-backed + // mounts return an empty list by design (discovery is per-alias), so + // this is typically just the local set. + let mut records = self.local.all_records().await?; + for mount in &self.mounts { + let remote = mount.lookup.all_records().await?; + records.extend(remote.into_iter().map(|r| mount.localize_record(r))); + } + Ok(records) + } + + async fn list_branches(&self, ledger_name: &str) -> Result> { + match self.mount_for(ledger_name) { + Some((mount, remote)) => Ok(mount + .lookup + .list_branches(remote) + .await? + .into_iter() + .map(|r| mount.localize_record(r)) + .collect()), + None => self.local.list_branches(ledger_name).await, + } + } +} + +#[async_trait] +impl GraphSourceLookup for CompositeNameService { + async fn lookup_graph_source( + &self, + graph_source_id: &str, + ) -> Result> { + match self.mount_for(graph_source_id) { + Some((mount, remote)) => Ok(mount + .lookup + .lookup_graph_source(remote) + .await? + .map(|r| mount.localize_graph_source(r))), + None => self.local.lookup_graph_source(graph_source_id).await, + } + } + + async fn lookup_any(&self, resource_id: &str) -> Result { + match self.mount_for(resource_id) { + Some((mount, remote)) => Ok(match mount.lookup.lookup_any(remote).await? { + NsLookupResult::Ledger(r) => NsLookupResult::Ledger(mount.localize_record(r)), + NsLookupResult::GraphSource(r) => { + NsLookupResult::GraphSource(mount.localize_graph_source(r)) + } + NsLookupResult::NotFound => NsLookupResult::NotFound, + }), + None => self.local.lookup_any(resource_id).await, + } + } + + async fn all_graph_source_records(&self) -> Result> { + let mut records = self.local.all_graph_source_records().await?; + for mount in &self.mounts { + let remote = mount.lookup.all_graph_source_records().await?; + records.extend(remote.into_iter().map(|r| mount.localize_graph_source(r))); + } + Ok(records) + } +} + +#[async_trait] +impl RefLookup for CompositeNameService { + async fn get_ref(&self, ledger_id: &str, kind: RefKind) -> Result> { + match self.mount_for(ledger_id) { + Some((mount, remote)) => mount.lookup.get_ref(remote, kind).await, + None => self.local.get_ref(ledger_id, kind).await, + } + } +} + +#[async_trait] +impl StatusLookup for CompositeNameService { + async fn get_status(&self, ledger_id: &str) -> Result> { + match self.mount_for(ledger_id) { + Some((mount, remote)) => mount.lookup.get_status(remote).await, + None => self.local.get_status(ledger_id).await, + } + } +} + +#[async_trait] +impl ConfigLookup for CompositeNameService { + async fn get_config(&self, ledger_id: &str) -> Result> { + match self.mount_for(ledger_id) { + Some((mount, remote)) => mount.lookup.get_config(remote).await, + None => self.local.get_config(ledger_id).await, + } + } +} + +// --------------------------------------------------------------------------- +// Write surface: reject mounted aliases, delegate the rest to local +// --------------------------------------------------------------------------- + +#[async_trait] +impl CommitPublisher for CompositeNameService { + async fn publish_commit( + &self, + ledger_id: &str, + commit_t: i64, + commit_id: &ContentId, + ) -> Result<()> { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local + .publish_commit(ledger_id, commit_t, commit_id) + .await + } + + fn publishing_ledger_id(&self, ledger_id: &str) -> Option { + if self.mount_for(ledger_id).is_some() { + return None; + } + self.local.publishing_ledger_id(ledger_id) + } +} + +#[async_trait] +impl IndexPublisher for CompositeNameService { + async fn publish_index( + &self, + ledger_id: &str, + index_t: i64, + index_id: &ContentId, + ) -> Result<()> { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local.publish_index(ledger_id, index_t, index_id).await + } +} + +#[async_trait] +impl LedgerLifecycle for CompositeNameService { + async fn init(&self, ledger_id: &str) -> Result<()> { + // Also prevents creating a local ledger that would shadow a mount. + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local.init(ledger_id).await + } + + async fn retract(&self, ledger_id: &str) -> Result<()> { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local.retract(ledger_id).await + } + + async fn purge(&self, ledger_id: &str) -> Result<()> { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local.purge(ledger_id).await + } +} + +#[async_trait] +impl BranchLifecycle for CompositeNameService { + async fn create_branch( + &self, + ledger_name: &str, + new_branch: &str, + source_branch: &str, + at_commit: Option<(ContentId, i64)>, + ) -> Result<()> { + if let Some(err) = self.reject_mounted_write(ledger_name) { + return Err(err); + } + self.local + .create_branch(ledger_name, new_branch, source_branch, at_commit) + .await + } + + async fn drop_branch(&self, ledger_id: &str) -> Result> { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local.drop_branch(ledger_id).await + } + + async fn reset_head(&self, ledger_id: &str, snapshot: NsRecordSnapshot) -> Result<()> { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local.reset_head(ledger_id, snapshot).await + } + + async fn pending_commit_cids( + &self, + ledger_id: &str, + since_t: i64, + ) -> Result>> { + if self.mount_for(ledger_id).is_some() { + // No commit-CID index for mounts; callers fall back to the DAG walk. + return Ok(None); + } + self.local.pending_commit_cids(ledger_id, since_t).await + } + + async fn prune_commit_index(&self, ledger_id: &str, up_to_t: i64) -> Result<()> { + if self.mount_for(ledger_id).is_some() { + return Ok(()); + } + self.local.prune_commit_index(ledger_id, up_to_t).await + } +} + +#[async_trait] +impl AdminPublisher for CompositeNameService { + async fn publish_index_allow_equal( + &self, + ledger_id: &str, + index_t: i64, + index_id: &ContentId, + ) -> Result<()> { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local + .publish_index_allow_equal(ledger_id, index_t, index_id) + .await + } +} + +#[async_trait] +impl RefPublisher for CompositeNameService { + async fn compare_and_set_ref( + &self, + ledger_id: &str, + kind: RefKind, + expected: Option<&RefValue>, + new: &RefValue, + ) -> Result { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local + .compare_and_set_ref(ledger_id, kind, expected, new) + .await + } +} + +#[async_trait] +impl GraphSourcePublisher for CompositeNameService { + async fn publish_graph_source( + &self, + name: &str, + branch: &str, + source_type: GraphSourceType, + config: &str, + dependencies: &[String], + ) -> Result<()> { + if let Some(err) = self.reject_mounted_write(name) { + return Err(err); + } + self.local + .publish_graph_source(name, branch, source_type, config, dependencies) + .await + } + + async fn publish_graph_source_index( + &self, + name: &str, + branch: &str, + index_id: &ContentId, + index_t: i64, + ) -> Result<()> { + if let Some(err) = self.reject_mounted_write(name) { + return Err(err); + } + self.local + .publish_graph_source_index(name, branch, index_id, index_t) + .await + } + + async fn retract_graph_source(&self, name: &str, branch: &str) -> Result<()> { + if let Some(err) = self.reject_mounted_write(name) { + return Err(err); + } + self.local.retract_graph_source(name, branch).await + } +} + +#[async_trait] +impl StatusPublisher for CompositeNameService { + async fn push_status( + &self, + ledger_id: &str, + expected: Option<&StatusValue>, + new: &StatusValue, + ) -> Result { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local.push_status(ledger_id, expected, new).await + } +} + +#[async_trait] +impl ConfigPublisher for CompositeNameService { + async fn push_config( + &self, + ledger_id: &str, + expected: Option<&ConfigValue>, + new: &ConfigValue, + ) -> Result { + if let Some(err) = self.reject_mounted_write(ledger_id) { + return Err(err); + } + self.local.push_config(ledger_id, expected, new).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::memory::MemoryNameService; + + fn mounted_composite() -> (Arc, CompositeNameService) { + let local = Arc::new(MemoryNameService::new()); + let remote = Arc::new(MemoryNameService::new()); + let composite = CompositeNameService::new( + local.clone(), + vec![RemoteMount::new("acme", remote.clone())], + ) + .expect("composite"); + (remote, composite) + } + + #[tokio::test] + async fn lookup_routes_by_prefix_and_localizes() { + let (remote, composite) = mounted_composite(); + remote.init("inventory:main").await.expect("init remote"); + + let record = composite + .lookup("acme/inventory:main") + .await + .expect("lookup") + .expect("mounted record found"); + assert_eq!(record.ledger_id, "acme/inventory:main"); + assert_eq!(record.name, "acme/inventory"); + assert_eq!(record.branch, "main"); + + // The local nameservice does not see mounted aliases. + assert!(composite + .lookup("inventory:main") + .await + .expect("local lookup") + .is_none()); + } + + #[tokio::test] + async fn local_aliases_route_to_local_publisher() { + let (_remote, composite) = mounted_composite(); + composite.init("books:main").await.expect("init local"); + let record = composite + .lookup("books:main") + .await + .expect("lookup") + .expect("local record"); + assert_eq!(record.ledger_id, "books:main"); + } + + #[tokio::test] + async fn writes_to_mounted_aliases_are_rejected() { + let (remote, composite) = mounted_composite(); + remote.init("inventory:main").await.expect("init remote"); + + let err = composite + .init("acme/other:main") + .await + .expect_err("init on mount must fail"); + assert!( + err.to_string().contains("read-only remote mount"), + "unexpected error: {err}" + ); + + let err = composite + .publish_commit( + "acme/inventory:main", + 1, + &ContentId::new(fluree_db_core::ContentKind::Commit, b"c"), + ) + .await + .expect_err("publish_commit on mount must fail"); + assert!(err.to_string().contains("read-only remote mount")); + } + + #[tokio::test] + async fn prefix_requires_separator() { + let (_remote, composite) = mounted_composite(); + // "acmecorp:main" starts with "acme" but is not under the mount. + composite.init("acmecorp:main").await.expect("local init"); + let record = composite + .lookup("acmecorp:main") + .await + .expect("lookup") + .expect("local record"); + assert_eq!(record.ledger_id, "acmecorp:main"); + } + + #[test] + fn duplicate_prefixes_rejected() { + let local = Arc::new(MemoryNameService::new()); + let remote = Arc::new(MemoryNameService::new()); + let result = CompositeNameService::new( + local, + vec![ + RemoteMount::new("acme", remote.clone()), + RemoteMount::new("acme", remote), + ], + ); + assert!(result.is_err()); + } +} diff --git a/fluree-db-server/src/peer/mod.rs b/fluree-db-server/src/peer/mod.rs index 48f049fa42..e16291e57e 100644 --- a/fluree-db-server/src/peer/mod.rs +++ b/fluree-db-server/src/peer/mod.rs @@ -27,7 +27,7 @@ pub mod sync_task; pub use forward::{ForwardingClient, ForwardingError}; pub use proxy_nameservice::ProxyNameService; -pub use proxy_storage::ProxyStorage; +pub use proxy_storage::{ProxyReadMode, ProxyStorage}; pub use state::{ GraphSourceNeedsRefresh, NeedsRefresh, PeerState, RemoteGraphSourceWatermark, RemoteLedgerWatermark, diff --git a/fluree-db-server/src/peer/proxy_nameservice.rs b/fluree-db-server/src/peer/proxy_nameservice.rs index 6e001c0636..5c4238c7c2 100644 --- a/fluree-db-server/src/peer/proxy_nameservice.rs +++ b/fluree-db-server/src/peer/proxy_nameservice.rs @@ -1,413 +1,8 @@ -//! Proxy nameservice implementation for peer mode +//! Re-export of the proxy nameservice client. //! -//! Fetches nameservice records via the transaction server's `/v1/fluree/storage/ns/{alias}` -//! endpoint instead of direct file access. This allows peers to operate without storage -//! credentials. +//! `ProxyNameService` moved to `fluree-db-nameservice-sync` so the CLI and +//! other consumers can build peer-mode/remote-mount Fluree instances without +//! depending on the server crate. This module keeps the historical +//! `fluree_db_server::peer::ProxyNameService` path working. -use async_trait::async_trait; -use fluree_db_nameservice::{NameServiceError, NsRecord, Result}; -use reqwest::{Client, StatusCode}; -use serde::Deserialize; -use std::fmt::Debug; -use std::time::Duration; - -/// NameService implementation that proxies lookups through the transaction server -#[derive(Clone)] -pub struct ProxyNameService { - client: Client, - base_url: String, - token: String, -} - -impl Debug for ProxyNameService { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ProxyNameService") - .field("base_url", &self.base_url) - .finish_non_exhaustive() - } -} - -/// Response from nameservice lookup endpoint. -/// Must match `NsRecordResponse` from routes/storage_proxy.rs. -/// -/// Uses `#[serde(default)]` on optional fields so that missing JSON keys -/// deserialize as `None` rather than failing — this keeps the peer -/// forward-compatible when the server adds new fields. -#[derive(Debug, Deserialize)] -struct NsRecordResponse { - #[serde(default)] - name: Option, - branch: String, - commit_head_id: Option, - commit_t: i64, - index_head_id: Option, - index_t: i64, - #[serde(default)] - default_context: Option, - retracted: bool, - #[serde(default)] - config_id: Option, - /// Parent branch this branch was forked from. Required for - /// peers to build the `BranchedContentStore` that resolves - /// commits inherited from the source branch — without it, all - /// reads of inherited commits 404 even when the ledger is - /// reachable. - #[serde(default)] - source_branch: Option, - /// Number of child branches forked from this one. Defaults to - /// 0 when an older server omits the field. - #[serde(default)] - branches: u32, -} - -impl NsRecordResponse { - /// Convert to NsRecord, using the original lookup key as the ledger_id. - /// - /// When the server omits `name`, derive it from `lookup_key` by splitting - /// on `:` (e.g., `"books:main"` → `"books"`). This avoids copying the full - /// `ledger_id` (which includes the branch) into the `name` field. - fn into_ns_record(self, lookup_key: &str) -> NsRecord { - use fluree_db_core::ContentId; - - let derived_name = self.name.unwrap_or_else(|| { - lookup_key - .split_once(':') - .map(|(name, _branch)| name.to_string()) - .unwrap_or_else(|| lookup_key.to_string()) - }); - - NsRecord { - // ledger_id is the key used for lookup (may differ from name) - ledger_id: lookup_key.to_string(), - name: derived_name, - branch: self.branch, - commit_head_id: self - .commit_head_id - .and_then(|s| s.parse::().ok()), - config_id: self.config_id.and_then(|s| s.parse::().ok()), - commit_t: self.commit_t, - index_head_id: self.index_head_id.and_then(|s| s.parse::().ok()), - index_t: self.index_t, - default_context: self - .default_context - .and_then(|s| s.parse::().ok()), - retracted: self.retracted, - source_branch: self.source_branch, - branches: self.branches, - } - } -} - -impl ProxyNameService { - /// Create a new proxy nameservice client - /// - /// # Arguments - /// - /// * `base_url` - Base URL of the transaction server (e.g., `https://tx.fluree.internal:8090`) - /// * `token` - Bearer token for authentication (with `fluree.storage.*` claims) - pub fn new(base_url: String, token: String) -> Self { - let client = Client::builder() - .timeout(Duration::from_secs(30)) // 30 seconds for NS lookups - .build() - .expect("Failed to create proxy nameservice client"); - - // Normalize base_url by trimming trailing slashes - let base_url = base_url.trim_end_matches('/').to_string(); - - Self { - client, - base_url, - token, - } - } - - /// Build the nameservice lookup endpoint URL - fn ns_url(&self, alias: &str) -> String { - format!( - "{}/v1/fluree/storage/ns/{}", - self.base_url, - urlencoding::encode(alias) - ) - } -} - -#[async_trait] -impl fluree_db_nameservice::RefLookup for ProxyNameService { - async fn get_ref( - &self, - _ledger_id: &str, - _kind: fluree_db_nameservice::RefKind, - ) -> Result> { - Err(NameServiceError::storage( - "get_ref not supported in proxy mode".to_string(), - )) - } -} - -#[async_trait] -impl fluree_db_nameservice::StatusLookup for ProxyNameService { - async fn get_status( - &self, - _ledger_id: &str, - ) -> Result> { - Err(NameServiceError::storage( - "get_status not supported in proxy mode".to_string(), - )) - } -} - -#[async_trait] -impl fluree_db_nameservice::ConfigLookup for ProxyNameService { - async fn get_config( - &self, - _ledger_id: &str, - ) -> Result> { - Err(NameServiceError::storage( - "get_config not supported in proxy mode".to_string(), - )) - } -} - -#[async_trait] -impl fluree_db_nameservice::NameServiceLookup for ProxyNameService { - async fn lookup(&self, ledger_id: &str) -> Result> { - let url = self.ns_url(ledger_id); - - let response = self - .client - .get(&url) - .header("Authorization", format!("Bearer {}", self.token)) - .send() - .await - .map_err(|e| { - NameServiceError::storage(format!("Nameservice proxy request failed: {e}")) - })?; - - let status = response.status(); - - match status { - StatusCode::OK => { - let ns_response: NsRecordResponse = response.json().await.map_err(|e| { - NameServiceError::storage(format!("Failed to parse NS response: {e}")) - })?; - Ok(Some(ns_response.into_ns_record(ledger_id))) - } - StatusCode::NOT_FOUND => Ok(None), - StatusCode::UNAUTHORIZED => Err(NameServiceError::storage(format!( - "Nameservice proxy authentication failed for {ledger_id}: check token validity" - ))), - StatusCode::FORBIDDEN => { - // Not in token scope - treat as not found (no existence leak) - Ok(None) - } - _ => Err(NameServiceError::storage(format!( - "Nameservice proxy unexpected status {status} for {ledger_id}" - ))), - } - } - - async fn all_records(&self) -> Result> { - // Peers use SSE for discovery, not all_records() - // Return empty - this is intentional for proxy mode - // The peer maintains its own view of known ledgers via SSE events - Ok(Vec::new()) - } -} - -// No `BranchLifecycle` impl for `ProxyNameService`: peer mode has no -// authority to mutate the nameservice. Branch lifecycle requests are -// served by the upstream transaction server (via its own HTTP routes, -// not through this trait). - -#[async_trait] -impl fluree_db_nameservice::GraphSourceLookup for ProxyNameService { - async fn lookup_graph_source( - &self, - _graph_source_id: &str, - ) -> Result> { - Ok(None) // Proxy doesn't have local graph source records - } - - async fn lookup_any(&self, resource_id: &str) -> Result { - // Delegate to the ledger lookup endpoint. Graph-source - // discovery isn't exposed through the storage proxy today, - // so a non-ledger resource still reports NotFound — but a - // real ledger record now resolves correctly instead of - // always returning NotFound and breaking every caller that - // routes through `GraphSourceLookup::lookup_any`. - use fluree_db_nameservice::{NameServiceLookup, NsLookupResult}; - match self.lookup(resource_id).await? { - Some(record) => Ok(NsLookupResult::Ledger(record)), - None => Ok(NsLookupResult::NotFound), - } - } - - async fn all_graph_source_records( - &self, - ) -> Result> { - Ok(Vec::new()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_proxy_nameservice_debug() { - let ns = ProxyNameService::new( - "http://localhost:8090".to_string(), - "test-token".to_string(), - ); - let debug = format!("{ns:?}"); - assert!(debug.contains("ProxyNameService")); - assert!(debug.contains("localhost:8090")); - // Token should NOT be in debug output - assert!(!debug.contains("test-token")); - } - - #[test] - fn test_ns_url() { - let ns = ProxyNameService::new( - "http://localhost:8090".to_string(), - "test-token".to_string(), - ); - assert_eq!( - ns.ns_url("books:main"), - "http://localhost:8090/v1/fluree/storage/ns/books%3Amain" - ); - } - - #[test] - fn test_ns_url_no_special_chars() { - let ns = ProxyNameService::new( - "http://localhost:8090".to_string(), - "test-token".to_string(), - ); - // Alias without colon doesn't need encoding - assert_eq!( - ns.ns_url("books"), - "http://localhost:8090/v1/fluree/storage/ns/books" - ); - } - - #[test] - fn test_ns_record_conversion() { - let response = NsRecordResponse { - name: Some("books".to_string()), - branch: "main".to_string(), - commit_head_id: None, - commit_t: 42, - index_head_id: None, - index_t: 40, - default_context: None, - retracted: false, - config_id: None, - source_branch: None, - branches: 0, - }; - - // Use the lookup key as ledger_id (simulating lookup("books")) - let record = response.into_ns_record("books"); - // ledger_id should be the lookup key, not the alias - assert_eq!(record.ledger_id, "books"); - assert_eq!(record.name, "books"); - assert_eq!(record.branch, "main"); - assert_eq!(record.commit_t, 42); - assert_eq!(record.index_t, 40); - assert!(!record.retracted); - // default_context is not exposed via proxy API - assert!(record.default_context.is_none()); - } - - /// Regression for finding #11: `source_branch` and `branches` - /// must round-trip through the proxy so peers can build the - /// `BranchedContentStore` for forked branches. Earlier code - /// hardcoded `source_branch: None` regardless of what the - /// server sent, breaking every read of an inherited commit. - #[test] - fn ns_record_conversion_preserves_branch_lineage() { - let response = NsRecordResponse { - name: Some("books".to_string()), - branch: "feature".to_string(), - commit_head_id: None, - commit_t: 7, - index_head_id: None, - index_t: 5, - default_context: None, - retracted: false, - config_id: None, - source_branch: Some("main".to_string()), - branches: 3, - }; - - let record = response.into_ns_record("books:feature"); - assert_eq!(record.source_branch.as_deref(), Some("main")); - assert_eq!(record.branches, 3); - } - - /// Old-server compatibility: when the wire response omits - /// `source_branch`/`branches`, deserialization defaults them - /// to `None`/`0` rather than failing. - #[test] - fn ns_record_response_deserializes_without_lineage_fields() { - let json = r#"{ - "name": "books", - "branch": "main", - "commit_head_id": null, - "commit_t": 0, - "index_head_id": null, - "index_t": 0, - "retracted": false - }"#; - let response: NsRecordResponse = - serde_json::from_str(json).expect("response without lineage fields parses"); - assert!(response.source_branch.is_none()); - assert_eq!(response.branches, 0); - } - - #[test] - fn test_ns_record_name_derived_from_lookup_key() { - // When server omits `name`, derive it by splitting lookup_key on ':' - let response = NsRecordResponse { - name: None, - branch: "main".to_string(), - commit_head_id: None, - commit_t: 10, - index_head_id: None, - index_t: 0, - default_context: None, - retracted: false, - config_id: None, - source_branch: None, - branches: 0, - }; - - let record = response.into_ns_record("books:main"); - assert_eq!(record.ledger_id, "books:main"); - // name should be "books", NOT "books:main" - assert_eq!(record.name, "books"); - } - - #[test] - fn test_ns_record_name_no_branch_in_lookup_key() { - // When lookup_key has no colon, use it as-is - let response = NsRecordResponse { - name: None, - branch: "main".to_string(), - commit_head_id: None, - commit_t: 10, - index_head_id: None, - index_t: 0, - default_context: None, - retracted: false, - config_id: None, - source_branch: None, - branches: 0, - }; - - let record = response.into_ns_record("books"); - assert_eq!(record.ledger_id, "books"); - assert_eq!(record.name, "books"); - } -} +pub use fluree_db_nameservice_sync::proxy_nameservice::ProxyNameService; diff --git a/fluree-db-server/src/peer/proxy_storage.rs b/fluree-db-server/src/peer/proxy_storage.rs index b72460fc28..40ec915d73 100644 --- a/fluree-db-server/src/peer/proxy_storage.rs +++ b/fluree-db-server/src/peer/proxy_storage.rs @@ -1,513 +1,8 @@ -//! Proxy storage implementation for peer mode +//! Re-export of the proxy storage client. //! -//! Fetches index data via the transaction server's `/v1/fluree/storage/block` endpoint -//! instead of direct storage access. This allows peers to operate without storage -//! credentials (no S3 access, no filesystem mount). -//! -//! ## Policy-Safe Reads -//! -//! Under `PolicyEnforced` mode (the only mode currently available via the storage -//! proxy), the server **always** returns decoded, policy-filtered flakes (FLKB format) -//! for leaf blocks — even if the client requests `application/octet-stream`. Raw FLI3 -//! leaf bytes are never returned to end users. -//! -//! Both `read_bytes()` and `read_bytes_hint()` use flakes-first content negotiation: -//! they request `application/x-fluree-flakes` first, falling back to -//! `application/octet-stream` on 406 (for non-leaf blocks like commits and branches). -//! -//! This means callers always receive: -//! - **Leaf blocks**: FLKB-encoded policy-filtered flakes -//! - **Non-leaf blocks**: raw bytes (commits, branches, manifests, etc.) -//! -//! The `ReadHint` distinction is preserved for forward-compatibility: if a -//! `TrustedInternal` enforcement mode is added later, `AnyBytes` could return -//! raw FLI3 leaves while `PreferLeafFlakes` would still prefer FLKB. - -use async_trait::async_trait; -use fluree_db_core::error::{Error as CoreError, Result}; -use fluree_db_core::format_ledger_id; -use fluree_db_core::storage::ReadHint; -use fluree_db_core::{ - ContentAddressedWrite, ContentId, ContentKind, ContentWriteResult, StorageRead, StorageWrite, - CODEC_FLUREE_COMMIT, CODEC_FLUREE_DICT_BLOB, CODEC_FLUREE_GARBAGE, CODEC_FLUREE_INDEX_BRANCH, - CODEC_FLUREE_INDEX_LEAF, CODEC_FLUREE_INDEX_ROOT, CODEC_FLUREE_LEDGER_CONFIG, CODEC_FLUREE_TXN, -}; -use reqwest::{Client, StatusCode}; -use serde::Serialize; -use std::fmt::Debug; -use std::time::Duration; - -/// Storage implementation that proxies reads through the transaction server -#[derive(Clone)] -pub struct ProxyStorage { - client: Client, - base_url: String, - token: String, -} - -impl Debug for ProxyStorage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ProxyStorage") - .field("base_url", &self.base_url) - .finish_non_exhaustive() - } -} - -/// Request body for the `/storage/block` endpoint. -/// -/// Both fields are required. The `cid` and `ledger` are derived from -/// the storage address via [`cid_and_ledger_from_address()`]. -#[derive(Debug, Serialize)] -struct BlockRequest { - cid: String, - ledger: String, -} - -/// Try to derive a CID string and ledger alias from a Fluree address. -/// -/// Parses the canonical `fluree:{method}://{alias_path}/{kind_dir}/{hash}.{ext}` -/// format produced by [`fluree_db_core::content_address()`]. -/// -/// Returns `(cid_string, ledger_alias)` on success, or `None` if the format -/// is unrecognized (in which case the caller should fall back to address-based). -fn cid_and_ledger_from_address(address: &str) -> Option<(String, String)> { - // Strip `fluree:{method}://` prefix - let rest = address.strip_prefix("fluree:")?; - let sep_pos = rest.find("://")?; - let path = &rest[sep_pos + 3..]; - - let parts: Vec<&str> = path.split('/').collect(); - let n = parts.len(); - if n < 3 { - return None; - } - - // Extract hash hex from the filename stem (before extension) - let filename = parts[n - 1]; - let (hash_hex, _ext) = filename.rsplit_once('.')?; - - // Determine the multicodec and where the alias ends based on directory structure. - // These patterns match those generated by `content_path()` in fluree-db-core. - let (codec, alias_end) = if n >= 2 && parts[n - 2] == "commit" { - (CODEC_FLUREE_COMMIT, n - 2) - } else if n >= 2 && parts[n - 2] == "txn" { - (CODEC_FLUREE_TXN, n - 2) - } else if n >= 2 && parts[n - 2] == "config" { - (CODEC_FLUREE_LEDGER_CONFIG, n - 2) - } else if n >= 3 && parts[n - 3] == "index" && parts[n - 2] == "roots" { - (CODEC_FLUREE_INDEX_ROOT, n - 3) - } else if n >= 3 && parts[n - 3] == "index" && parts[n - 2] == "garbage" { - (CODEC_FLUREE_GARBAGE, n - 3) - } else if n >= 4 && parts[n - 4] == "index" && parts[n - 3] == "objects" { - match parts[n - 2] { - "branches" => (CODEC_FLUREE_INDEX_BRANCH, n - 4), - "leaves" => (CODEC_FLUREE_INDEX_LEAF, n - 4), - _ => return None, - } - } else if n >= 3 && parts[n - 2] == "dicts" && parts[n - 3] == "@shared" { - // DictBlob: {name}/@shared/dicts/{hash}.{ext} - // The alias_end points before "@shared"; the ledger name is everything - // before it and we'll use the default branch below. - (CODEC_FLUREE_DICT_BLOB, n - 3) - } else { - return None; - }; - - // Reconstruct ledger ID from the alias segments before the kind directory. - // DictBlob uses @shared (no branch in path) — use the default branch. - // All other kinds have "name/.../branch/kind_dir/file" structure. - let ledger_id = if codec == CODEC_FLUREE_DICT_BLOB { - if alias_end < 1 { - return None; - } - let name = parts[..alias_end].join("/"); - format_ledger_id(&name, fluree_db_core::DEFAULT_BRANCH) - } else { - if alias_end < 2 { - return None; - } - let branch = parts[alias_end - 1]; - let name = parts[..alias_end - 1].join("/"); - format_ledger_id(&name, branch) - }; - - // Build CID from codec + hex digest - let cid = ContentId::from_hex_digest(codec, hash_hex)?; - Some((cid.to_string(), ledger_id)) -} - -/// Internal result type for fetch operations -/// -/// This avoids using CoreError for the 406 case, which is an internal -/// retry condition rather than a user-facing error. -enum FetchOutcome { - /// Successfully fetched bytes - Success(Vec), - /// Server returned 406 Not Acceptable (format not available) - NotAcceptable, - /// Fetch failed with an error - Error(CoreError), -} - -impl ProxyStorage { - /// Create a new proxy storage client - /// - /// # Arguments - /// - /// * `base_url` - Base URL of the transaction server (e.g., `https://tx.fluree.internal:8090`) - /// * `token` - Bearer token for authentication (with `fluree.storage.*` claims) - pub fn new(base_url: String, token: String) -> Self { - let client = Client::builder() - .timeout(Duration::from_secs(60)) // 1 minute for block reads - .build() - .expect("Failed to create proxy storage client"); - - // Normalize base_url by trimming trailing slashes - let base_url = base_url.trim_end_matches('/').to_string(); - - Self { - client, - base_url, - token, - } - } - - /// Build the storage block endpoint URL - fn block_url(&self) -> String { - format!("{}/v1/fluree/storage/block", self.base_url) - } - - /// Fetch with flakes-first content negotiation - /// - /// Tries `application/x-fluree-flakes` first. If the server returns 406 - /// (format not available for this block type), falls back to raw bytes. - async fn fetch_prefer_flakes(&self, address: &str) -> Result> { - // Try flakes format first - match self - .fetch_with_accept(address, "application/x-fluree-flakes") - .await - { - FetchOutcome::Success(bytes) => Ok(bytes), - FetchOutcome::NotAcceptable => { - // 406 = not a leaf or policy filtering not applicable - // Fall back to raw bytes - tracing::debug!( - address = %address, - "Flakes format not available, falling back to raw bytes" - ); - match self - .fetch_with_accept(address, "application/octet-stream") - .await - { - FetchOutcome::Success(bytes) => Ok(bytes), - FetchOutcome::NotAcceptable => { - // Should not happen for octet-stream, but handle anyway - Err(CoreError::storage(format!( - "Storage proxy: octet-stream also rejected for {address}" - ))) - } - FetchOutcome::Error(e) => Err(e), - } - } - FetchOutcome::Error(e) => Err(e), - } - } - - /// Internal fetch with a specific Accept header - /// - /// Returns `FetchOutcome` to distinguish between success, 406, and errors - /// without using string-based error detection. - /// - /// Derives CID+ledger from the address. Failure to parse is a hard error - /// (all storage addresses in the system use the canonical format). - async fn fetch_with_accept(&self, address: &str, accept: &str) -> FetchOutcome { - let url = self.block_url(); - let (cid, ledger) = match cid_and_ledger_from_address(address) { - Some(pair) => pair, - None => { - return FetchOutcome::Error(CoreError::storage(format!( - "Cannot derive CID from address: {address}" - ))); - } - }; - let body = BlockRequest { cid, ledger }; - - let response = match self - .client - .post(&url) - .header("Authorization", format!("Bearer {}", self.token)) - .header("Accept", accept) - .json(&body) - .send() - .await - { - Ok(r) => r, - Err(e) => { - let err = if e.is_timeout() { - CoreError::io(format!("Storage proxy timeout for {address}: {e}")) - } else if e.is_connect() { - CoreError::io(format!( - "Storage proxy connection failed for {address}: {e}" - )) - } else { - CoreError::io(format!("Storage proxy request failed for {address}: {e}")) - }; - return FetchOutcome::Error(err); - } - }; - - let status = response.status(); - - match status { - StatusCode::OK => match response.bytes().await { - Ok(bytes) => FetchOutcome::Success(bytes.to_vec()), - Err(e) => FetchOutcome::Error(CoreError::io(format!( - "Failed to read response body for {address}: {e}" - ))), - }, - StatusCode::NOT_FOUND => FetchOutcome::Error(CoreError::not_found(address)), - StatusCode::NOT_ACCEPTABLE => { - // 406 - format not available, signal for retry with different Accept - FetchOutcome::NotAcceptable - } - StatusCode::UNAUTHORIZED => FetchOutcome::Error(CoreError::storage(format!( - "Storage proxy authentication failed for {address}: check token validity" - ))), - StatusCode::FORBIDDEN => { - // Address not in token scope - treat as not found (no existence leak) - FetchOutcome::Error(CoreError::not_found(address)) - } - s if s.is_server_error() => FetchOutcome::Error(CoreError::io(format!( - "Storage proxy server error for {address}: {status}" - ))), - _ => FetchOutcome::Error(CoreError::storage(format!( - "Storage proxy unexpected status {status} for {address}" - ))), - } - } -} - -#[async_trait] -impl StorageRead for ProxyStorage { - async fn read_bytes(&self, address: &str) -> Result> { - // Under PolicyEnforced, leaf blocks always return FLKB (not raw FLI3). - // Use flakes-first negotiation for deterministic behavior across block types: - // - Leaves → FLKB (policy-filtered flakes) - // - Non-leaves → raw bytes (via 406 fallback to octet-stream) - self.fetch_prefer_flakes(address).await - } - - // read_byte_range: uses default (full read + slice). ProxyStorage's HTTP - // protocol doesn't support Range headers yet — the default fallback is - // correct and safe. Future optimization: pass Range header through proxy. - - async fn read_bytes_hint(&self, address: &str, hint: ReadHint) -> Result> { - // Under PolicyEnforced, both AnyBytes and PreferLeafFlakes produce the same - // result (flakes-first negotiation). The distinction is preserved for - // forward-compatibility with TrustedInternal mode. - match hint { - ReadHint::AnyBytes => self.read_bytes(address).await, - ReadHint::PreferLeafFlakes => self.fetch_prefer_flakes(address).await, - // Future ReadHint variants fall back to default - _ => self.read_bytes(address).await, - } - } - - async fn exists(&self, address: &str) -> Result { - // In v1, implement exists as try-read - // This is correct but slightly less efficient than a HEAD request - // The server currently only supports POST for blocks anyway - match self.read_bytes(address).await { - Ok(_) => Ok(true), - Err(CoreError::NotFound(_)) => Ok(false), - Err(e) => Err(e), - } - } - - async fn list_prefix(&self, _prefix: &str) -> Result> { - // ProxyStorage is read-only and doesn't support listing - Err(CoreError::storage( - "ProxyStorage does not support list_prefix".to_string(), - )) - } -} - -#[async_trait] -impl StorageWrite for ProxyStorage { - async fn write_bytes(&self, _address: &str, _bytes: &[u8]) -> Result<()> { - Err(CoreError::storage( - "ProxyStorage is read-only (writes must go to the transaction server)".to_string(), - )) - } - - async fn delete(&self, _address: &str) -> Result<()> { - Err(CoreError::storage( - "ProxyStorage is read-only (deletes must go to the transaction server)".to_string(), - )) - } -} - -#[async_trait] -impl ContentAddressedWrite for ProxyStorage { - async fn content_write_bytes_with_hash( - &self, - _kind: ContentKind, - _ledger_alias: &str, - _content_hash_hex: &str, - _bytes: &[u8], - ) -> Result { - Err(CoreError::storage( - "ProxyStorage is read-only (writes must go to the transaction server)".to_string(), - )) - } - - async fn content_write_bytes( - &self, - _kind: ContentKind, - _ledger_alias: &str, - _bytes: &[u8], - ) -> Result { - Err(CoreError::storage( - "ProxyStorage is read-only (writes must go to the transaction server)".to_string(), - )) - } -} - -impl fluree_db_core::StorageMethod for ProxyStorage { - fn storage_method(&self) -> &'static str { - "proxy" - } -} - -#[cfg(test)] -mod tests { - use super::*; - use fluree_db_core::ContentKind; - - #[test] - fn test_proxy_storage_debug() { - let storage = ProxyStorage::new( - "http://localhost:8090".to_string(), - "test-token".to_string(), - ); - let debug = format!("{storage:?}"); - assert!(debug.contains("ProxyStorage")); - assert!(debug.contains("localhost:8090")); - // Token should NOT be in debug output - assert!(!debug.contains("test-token")); - } - - #[test] - fn test_block_url() { - let storage = ProxyStorage::new( - "http://localhost:8090".to_string(), - "test-token".to_string(), - ); - assert_eq!( - storage.block_url(), - "http://localhost:8090/v1/fluree/storage/block" - ); - } - - #[test] - fn test_block_url_with_trailing_slash() { - let storage = ProxyStorage::new( - "http://localhost:8090/".to_string(), - "test-token".to_string(), - ); - // Should work but might have double slash - that's okay for URLs - assert!(storage.block_url().contains("/v1/fluree/storage/block")); - } - - // ======================================================================== - // cid_and_ledger_from_address tests - // ======================================================================== - - /// Round-trip helper: build an address via `content_address`, then verify - /// `cid_and_ledger_from_address` recovers the correct CID and ledger. - fn assert_roundtrip(kind: ContentKind, alias: &str, data: &[u8]) { - let id = ContentId::new(kind, data); - let address = fluree_db_core::content_address("file", kind, alias, &id.digest_hex()); - let (cid_str, ledger) = - cid_and_ledger_from_address(&address).expect("should parse address"); - assert_eq!(cid_str, id.to_string(), "CID mismatch for {address}"); - assert_eq!(ledger, alias, "ledger mismatch for {address}"); - } - - #[test] - fn test_parse_commit_address() { - assert_roundtrip(ContentKind::Commit, "mydb:main", b"commit data"); - } - - #[test] - fn test_parse_txn_address() { - assert_roundtrip(ContentKind::Txn, "mydb:main", b"txn data"); - } - - #[test] - fn test_parse_index_root_address() { - assert_roundtrip(ContentKind::IndexRoot, "mydb:main", b"root data"); - } - - #[test] - fn test_parse_index_branch_address() { - assert_roundtrip(ContentKind::IndexBranch, "mydb:main", b"branch data"); - } - - #[test] - fn test_parse_index_leaf_address() { - assert_roundtrip(ContentKind::IndexLeaf, "mydb:main", b"leaf data"); - } - - #[test] - fn test_parse_dict_blob_address() { - use fluree_db_core::DictKind; - assert_roundtrip( - ContentKind::DictBlob { - dict: DictKind::Graphs, - }, - "mydb:main", - b"dict data", - ); - } - - #[test] - fn test_parse_config_address() { - assert_roundtrip(ContentKind::LedgerConfig, "mydb:main", b"config data"); - } - - #[test] - fn test_parse_garbage_address() { - assert_roundtrip(ContentKind::GarbageRecord, "mydb:main", b"gc data"); - } - - #[test] - fn test_parse_address_with_s3_method() { - let id = ContentId::new(ContentKind::Commit, b"s3 test"); - let address = fluree_db_core::content_address( - "s3", - ContentKind::Commit, - "prod:main", - &id.digest_hex(), - ); - let (cid_str, ledger) = - cid_and_ledger_from_address(&address).expect("should parse s3 address"); - assert_eq!(cid_str, id.to_string()); - assert_eq!(ledger, "prod:main"); - } - - #[test] - fn test_parse_address_not_fluree() { - assert!(cid_and_ledger_from_address("https://example.com/foo").is_none()); - } - - #[test] - fn test_parse_address_too_short() { - assert!(cid_and_ledger_from_address("fluree:file://a/b").is_none()); - } +//! `ProxyStorage` moved to `fluree-db-nameservice-sync` so the CLI and other +//! consumers can build peer-mode/remote-mount Fluree instances without +//! depending on the server crate. This module keeps the historical +//! `fluree_db_server::peer::ProxyStorage` path working. - #[test] - fn test_parse_address_unknown_kind_dir() { - assert!(cid_and_ledger_from_address("fluree:file://mydb/main/unknown/abc.bin").is_none()); - } -} +pub use fluree_db_nameservice_sync::proxy_storage::{ProxyReadMode, ProxyStorage}; diff --git a/fluree-db-server/src/routes/admin.rs b/fluree-db-server/src/routes/admin.rs index b3d4878af9..ce0a2b7886 100644 --- a/fluree-db-server/src/routes/admin.rs +++ b/fluree-db-server/src/routes/admin.rs @@ -315,6 +315,17 @@ pub async fn discovery(State(state): State>) -> Json) -> Self { + Self { + query: cfg.and_then(|s| s.serve_query).unwrap_or(true), + blocks: cfg.and_then(|s| s.serve_blocks).unwrap_or(true), + public: cfg.and_then(|s| s.public_visibility).unwrap_or(false), + } + } + + /// Serving tiers as advertised in nameservice responses. + pub fn advertised(&self) -> Vec<&'static str> { + let mut tiers = Vec::with_capacity(2); + if self.query { + tiers.push("query"); + } + if self.blocks { + tiers.push("blocks"); + } + tiers + } +} + +/// Resolve the serving posture from an already-loaded ledger state. +/// +/// Reads the config graph as-of `state.t()` (novelty-inclusive), so a +/// committed-but-unindexed config change takes effect immediately. +pub(crate) async fn effective_serving_from_state( + state: &LedgerState, +) -> Result { + let overlay: &dyn OverlayProvider = &*state.novelty; + let config = config_resolver::resolve_ledger_config(&state.snapshot, overlay, state.t()) + .await + .map_err(|e| ServerError::internal(format!("Serving config resolution failed: {e}")))?; + Ok(EffectiveServing::from_config( + config.as_ref().and_then(|c| c.serving.as_ref()), + )) +} + +/// Resolve the serving posture for a ledger by alias. +/// +/// Loads (or reuses) the cached ledger handle. Callers on anti-leak paths +/// should map load failures to their endpoint's 404 convention. +pub(crate) async fn effective_serving( + fluree: &Fluree, + ledger_id: &str, +) -> Result { + let handle = fluree + .ledger_cached(ledger_id) + .await + .map_err(ServerError::Api)?; + let state = handle.snapshot().await.to_ledger_state(); + effective_serving_from_state(&state).await +} diff --git a/fluree-db-server/src/routes/storage_proxy.rs b/fluree-db-server/src/routes/storage_proxy.rs index b170a43fc2..2371cff59c 100644 --- a/fluree-db-server/src/routes/storage_proxy.rs +++ b/fluree-db-server/src/routes/storage_proxy.rs @@ -206,6 +206,12 @@ pub struct NsRecordResponse { /// invariant. #[serde(skip_serializing_if = "is_zero")] pub branches: u32, + /// Serving tiers this server offers for the ledger (`"query"`, + /// `"blocks"`), computed from the ledger's `f:servingDefaults`. + /// Clients use this to pick between query-shipping and peer (local + /// compute) modes. Omitted when the posture could not be resolved. + #[serde(skip_serializing_if = "Option::is_none")] + pub serving: Option>, } fn is_zero(v: &u32) -> bool { @@ -224,6 +230,74 @@ pub struct BlockRequest { pub ledger: String, } +// ============================================================================ +// Range Requests +// ============================================================================ + +/// Parse a single-range `Range: bytes=start-end` header into a half-open +/// byte range. Returns `None` for absent, malformed, multi-range, or +/// suffix-form (`bytes=-N`) headers — callers fall back to a full 200 +/// response, which is always a valid way to satisfy a Range request. +fn parse_range_header(headers: &HeaderMap) -> Option> { + let value = headers.get(header::RANGE)?.to_str().ok()?; + let spec = value.strip_prefix("bytes=")?; + if spec.contains(',') { + return None; + } + let (start_s, end_s) = spec.split_once('-')?; + let start: u64 = start_s.trim().parse().ok()?; + let end = if end_s.trim().is_empty() { + u64::MAX + } else { + // HTTP ranges are inclusive; convert to half-open. + end_s.trim().parse::().ok()?.checked_add(1)? + }; + (start < end).then_some(start..end) +} + +// ============================================================================ +// Ledger Resolution +// ============================================================================ + +/// Resolve the ledger a block/object request refers to. +/// +/// Acts as the namespace guard (graph-source aliases and unknown ledgers +/// resolve to `None`) and normalizes the alias to the canonical `ledger_id`. +/// +/// Dict blobs need special handling: their `@shared` addresses carry only the +/// ledger *name*, so proxy clients derive a default-branch alias that may not +/// exist when the ledger lives on another branch. For dict-blob CIDs, fall +/// back to any live branch of the same name — dict content is name-scoped, +/// so any branch is equivalent for authorization and address derivation. +async fn resolve_block_ledger( + fluree: &fluree_db_api::Fluree, + kind: ContentKind, + ledger: &str, +) -> Result, ServerError> { + let ns = fluree.nameservice(); + if let Some(record) = ns + .lookup(ledger) + .await + .map_err(|e| ServerError::internal(format!("Nameservice lookup failed: {e}")))? + { + return Ok(Some(record.ledger_id)); + } + + if matches!(kind, ContentKind::DictBlob { .. }) { + if let Ok((name, _)) = fluree_db_core::ledger_id::split_ledger_id(ledger) { + let branches = ns + .list_branches(&name) + .await + .map_err(|e| ServerError::internal(format!("Branch listing failed: {e}")))?; + if let Some(record) = branches.into_iter().next() { + return Ok(Some(record.ledger_id)); + } + } + } + + Ok(None) +} + // ============================================================================ // Handlers // ============================================================================ @@ -259,6 +333,17 @@ pub async fn get_ns_record( .map_err(|e| ServerError::internal(format!("Nameservice lookup failed: {e}")))? .ok_or_else(|| ServerError::not_found("Ledger not found"))?; + // Advertise the serving tiers this server offers for the ledger. + // Best-effort: a ledger that fails to load can't resolve its posture, + // and the record lookup itself should still succeed. + let serving = match crate::routes::serving::effective_serving(&state.fluree, &ledger_id).await { + Ok(s) => Some(s.advertised()), + Err(e) => { + tracing::warn!(ledger_id, error = %e, "serving posture unresolved; omitting from NS record"); + None + } + }; + Ok(Json(NsRecordResponse { // IMPORTANT: this endpoint is consumed by `fluree-db-nameservice-sync` which // deserializes into `NsRecord`. Therefore we must include all required @@ -287,6 +372,7 @@ pub async fn get_ns_record( .map(std::string::ToString::to_string), source_branch: ns_record.source_branch.clone(), branches: ns_record.branches, + serving, })) } @@ -329,21 +415,16 @@ pub async fn get_block( .content_kind() .filter(|k| block_fetch::is_allowed_block_kind(*k)) .ok_or_else(|| ServerError::not_found("Block not found"))?; - // Suppress unused variable warning — `kind` validated above, address derived by - // `fetch_and_decode_block` internally. - let _ = kind; - // 3. Namespace guard: ensure `body.ledger` is a real ledger (not a graph source alias) + // 3. Namespace guard: resolve `body.ledger` to a real ledger (not a graph + // source alias); dict-blob requests may need branch resolution. let fluree = &state.fluree; - fluree - .nameservice() - .lookup(&body.ledger) - .await - .map_err(|e| ServerError::internal(format!("Nameservice lookup failed: {e}")))? + let effective_ledger = resolve_block_ledger(fluree, kind, &body.ledger) + .await? .ok_or_else(|| ServerError::not_found("Block not found"))?; // 4. Authorize: token scope must include this ledger - block_fetch::authorize_ledger(&principal.to_block_access_scope(), &body.ledger) + block_fetch::authorize_ledger(&principal.to_block_access_scope(), &effective_ledger) .map_err(|_| ServerError::not_found("Block not found"))?; // Get storage proxy config for defaults and debug headers @@ -367,14 +448,24 @@ pub async fn get_block( // Load ledger context for leaf decoding + policy filtering // Force a fresh load so policy evaluation sees current data. // This avoids stale-cache issues after reindex updates. - fluree.disconnect_ledger(&body.ledger).await; + fluree.disconnect_ledger(&effective_ledger).await; let handle = fluree - .ledger_cached(&body.ledger) + .ledger_cached(&effective_ledger) .await .map_err(|e| ServerError::internal(format!("Ledger load failed: {e}")))?; let snapshot = handle.snapshot().await; let to_t = snapshot.snapshot.t; + + // Serving gate: honor the ledger's f:serveBlocks posture (404 per the + // no-existence-leak convention on this endpoint). + let serving = crate::routes::serving::effective_serving_from_state( + &handle.snapshot().await.to_ledger_state(), + ) + .await?; + if !serving.blocks { + return Err(ServerError::not_found("Block not found")); + } let ledger_ctx = LedgerBlockContext { snapshot: &snapshot.snapshot, to_t, @@ -388,7 +479,7 @@ pub async fn get_block( .ok_or_else(|| ServerError::internal("block fetch requires a managed storage backend"))?; let fetched = block_fetch::fetch_and_decode_block( &admin_storage, - &body.ledger, + &effective_ledger, &cid, Some(&ledger_ctx), &mode, @@ -502,19 +593,30 @@ fn verify_object_integrity(id: &ContentId, bytes: &[u8]) -> bool { /// # Query Parameters /// - `ledger`: Ledger alias (required, e.g., `"mydb:main"`) /// +/// # Range Requests +/// +/// A single-range `Range: bytes=start-end` header returns 206 Partial +/// Content with a `Content-Range` header. Integrity is verified against the +/// **full** object before slicing, so partial responses carry the same +/// corruption guarantee as full ones. Unsupported Range forms fall back to a +/// full 200 response; a start at or past the object length returns 416. +/// /// # Response Headers /// - `Content-Type: application/octet-stream` /// - `X-Fluree-Content-Kind`: content kind label (commit, txn, config, index-root, etc.) +/// - `Content-Range` (206 responses) /// /// # Errors /// - 400: Invalid CID string /// - 404: Object not found, disallowed kind, or not authorized +/// - 416: Range start beyond object length /// - 500: Hash verification failed (storage corruption) pub async fn get_object_by_cid( State(state): State>, Path(cid_str): Path, Query(query): Query, StorageProxyBearer(principal): StorageProxyBearer, + headers: HeaderMap, ) -> Result { // 1. Parse CID let id: ContentId = cid_str @@ -528,8 +630,22 @@ pub async fn get_object_by_cid( _ => return Err(ServerError::not_found("Object not found")), }; - // 3. Authorize: principal must have access to this ledger - if !principal.is_authorized_for_ledger(&query.ledger) { + // 3. Namespace guard: resolve `ledger` to a real ledger (not a graph + // source alias); dict-blob requests may need branch resolution. + let effective_ledger = resolve_block_ledger(&state.fluree, kind, &query.ledger) + .await? + .ok_or_else(|| ServerError::not_found("Object not found"))?; + + // 3b. Authorize: principal must have access to the resolved ledger. + if !principal.is_authorized_for_ledger(&effective_ledger) { + return Err(ServerError::not_found("Object not found")); + } + + // 3c. Serving gate: the ledger's f:serveBlocks posture must allow raw + // content serving. + let serving = + crate::routes::serving::effective_serving(&state.fluree, &effective_ledger).await?; + if !serving.blocks { return Err(ServerError::not_found("Object not found")); } @@ -540,7 +656,8 @@ pub async fn get_object_by_cid( .admin_storage_cloned() .ok_or_else(|| ServerError::internal("object fetch requires a managed storage backend"))?; let method = admin_storage.storage_method(); - let address = fluree_db_core::content_address(method, kind, &query.ledger, &id.digest_hex()); + let address = + fluree_db_core::content_address(method, kind, &effective_ledger, &id.digest_hex()); let bytes = admin_storage .read_bytes(&address) @@ -550,15 +667,39 @@ pub async fn get_object_by_cid( other => ServerError::internal(format!("Storage read: {other}")), })?; - // 5. Verify integrity (format-sniffing for commits) + // 5. Verify integrity (format-sniffing for commits). Always against the + // full object, so ranged responses inherit the guarantee. if !verify_object_integrity(&id, &bytes) { return Err(ServerError::internal(format!( "Hash verification failed for CID {cid_str}" ))); } - // 6. Build response + // 6. Build response (full or ranged) let kind_label = kind.codec_dir_name(); + if let Some(range) = parse_range_header(&headers) { + let total = bytes.len() as u64; + if range.start >= total { + return Response::builder() + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(header::CONTENT_RANGE, format!("bytes */{total}")) + .body(Body::empty()) + .map_err(|e| ServerError::internal(e.to_string())); + } + let end = range.end.min(total); + let slice = bytes[range.start as usize..end as usize].to_vec(); + return Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header(header::CONTENT_TYPE, "application/octet-stream") + .header("X-Fluree-Content-Kind", kind_label) + .header( + header::CONTENT_RANGE, + format!("bytes {}-{}/{total}", range.start, end - 1), + ) + .body(Body::from(slice)) + .map_err(|e| ServerError::internal(e.to_string())); + } + Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/octet-stream") diff --git a/fluree-db-server/src/state.rs b/fluree-db-server/src/state.rs index 8a832ed208..b38a873831 100644 --- a/fluree-db-server/src/state.rs +++ b/fluree-db-server/src/state.rs @@ -16,7 +16,7 @@ //! - **Proxy**: All storage reads proxied through transaction server (no storage credentials) use crate::config::{ServerConfig, ServerRole}; -use crate::peer::{ForwardingClient, PeerState, ProxyNameService, ProxyStorage}; +use crate::peer::{ForwardingClient, PeerState, ProxyNameService, ProxyReadMode, ProxyStorage}; use crate::registry::LedgerRegistry; use crate::telemetry::TelemetryConfig; use dashmap::DashMap; @@ -437,7 +437,11 @@ fn build_proxy_fluree( fluree_db_api::ApiError::internal(format!("Failed to load storage proxy token: {e}")) })?; - let storage = ProxyStorage::new(tx_url.clone(), token.clone()); + // Raw mode: the peer's `fluree.storage.*` token grants full read access, + // so it fetches canonical CAS bytes (CID-verified client-side). This is + // also what lets the binary index reader consume FLI3 leaves directly — + // the filtered (FLKB) tier has no leaf decoder on the read path. + let storage = ProxyStorage::new(tx_url.clone(), token.clone(), ProxyReadMode::Raw); let nameservice = ProxyNameService::new(tx_url, token); let ns_mode = NameServiceMode::ReadOnly(Arc::new(nameservice)); diff --git a/fluree-db-server/tests/proxy_integration.rs b/fluree-db-server/tests/proxy_integration.rs index 87c0da0da4..988ca2374b 100644 --- a/fluree-db-server/tests/proxy_integration.rs +++ b/fluree-db-server/tests/proxy_integration.rs @@ -1409,7 +1409,7 @@ async fn test_block_content_negotiation_returns_flkb_for_leaf() { #[tokio::test] async fn test_proxy_storage_read_bytes_hint_returns_flkb_for_leaf() { use fluree_db_core::ReadHint; - use fluree_db_server::peer::ProxyStorage; + use fluree_db_server::peer::{ProxyReadMode, ProxyStorage}; use tokio::net::TcpListener; // Create tx server state with storage proxy enabled @@ -1513,7 +1513,7 @@ async fn test_proxy_storage_read_bytes_hint_returns_flkb_for_leaf() { let leaf_address = leaf_address_from_cid(&leaf_cid, "peer:test"); // Create ProxyStorage pointing to our test server - let proxy_storage = ProxyStorage::new(server_url.clone(), token); + let proxy_storage = ProxyStorage::new(server_url.clone(), token, ProxyReadMode::Filtered); // Call read_bytes_hint with PreferLeafFlakes let result = proxy_storage @@ -1550,19 +1550,18 @@ async fn test_proxy_storage_read_bytes_hint_returns_flkb_for_leaf() { server_handle.abort(); } -/// Test that ProxyStorage.read_bytes returns FLKB for leaf blocks under PolicyEnforced +/// Test that ProxyStorage.read_bytes returns FLKB for leaf blocks in Filtered mode /// -/// Under PolicyEnforced mode (the only mode currently available via storage proxy), -/// leaf blocks are always decoded and policy-filtered. ProxyStorage.read_bytes() uses +/// In `ProxyReadMode::Filtered`, leaf blocks fetched via `/storage/block` are +/// always decoded and policy-filtered. ProxyStorage.read_bytes() uses /// flakes-first content negotiation, so leaves come back as FLKB (not raw FLI3). /// -/// Raw FLI3 leaf bytes would only be available under TrustedInternal enforcement mode, -/// which is not yet implemented. When it is, a separate ProxyStorage variant (or mode) -/// would be needed to opt into raw bytes. +/// Raw FLI3 leaf bytes are available via `ProxyReadMode::Raw`, which fetches +/// canonical CAS bytes through `/storage/objects/{cid}` instead. #[tokio::test] async fn test_proxy_storage_read_bytes_leaf_returns_flkb_under_policy() { use fluree_db_api::ReindexOptions; - use fluree_db_server::peer::ProxyStorage; + use fluree_db_server::peer::{ProxyReadMode, ProxyStorage}; use tokio::net::TcpListener; // Create tx server state with storage proxy enabled @@ -1659,7 +1658,7 @@ async fn test_proxy_storage_read_bytes_leaf_returns_flkb_under_policy() { let leaf_address = leaf_address_from_cid(&leaf_cid, "raw:test"); // Create ProxyStorage pointing to our test server - let proxy_storage = ProxyStorage::new(server_url.clone(), token); + let proxy_storage = ProxyStorage::new(server_url.clone(), token, ProxyReadMode::Filtered); // Call read_bytes (no hint) — under PolicyEnforced, this uses flakes-first // negotiation and returns FLKB for leaf blocks. @@ -1684,6 +1683,732 @@ async fn test_proxy_storage_read_bytes_leaf_returns_flkb_under_policy() { server_handle.abort(); } +/// Test that ProxyStorage in Raw mode returns canonical, CID-verified bytes +/// for both leaf and non-leaf blocks. +/// +/// Raw mode fetches through `GET /storage/objects/{cid}` (full-access tier): +/// - leaf bytes must be byte-identical to what's in the origin's storage +/// (raw FLI3, NOT the FLKB transport format), so the binary index reader +/// can consume them directly +/// - non-leaf bytes (index root) must round-trip identically too +/// - an out-of-scope token must observe NotFound (no existence leak) +#[tokio::test] +async fn test_proxy_storage_raw_mode_returns_canonical_bytes() { + use fluree_db_api::ReindexOptions; + use fluree_db_server::peer::{ProxyReadMode, ProxyStorage}; + use tokio::net::TcpListener; + + let (_tmp, state) = tx_server_state().await; + let app = build_router(state.clone()); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind to ephemeral port"); + let server_addr = listener.local_addr().expect("get local addr"); + let server_url = format!("http://{server_addr}"); + let server_handle = tokio::spawn(async move { + axum::serve(listener, app).await.expect("server run"); + }); + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + + let secret = [0u8; 32]; + let signing_key = SigningKey::from_bytes(&secret); + let token = create_storage_proxy_token_no_identity(&signing_key, true); + + let client = reqwest::Client::new(); + let create_resp = client + .post(format!("{server_url}/v1/fluree/create")) + .header("content-type", "application/json") + .body(r#"{"ledger": "rawmode:test"}"#) + .send() + .await + .expect("create ledger request"); + assert_eq!(create_resp.status(), reqwest::StatusCode::CREATED); + + let transact_resp = client + .post(format!("{server_url}/v1/fluree/update")) + .header("content-type", "application/json") + .body( + serde_json::json!({ + "ledger": "rawmode:test", + "@context": { "ex": "http://example.org/ns/" }, + "insert": { + "@graph": [ + { "@id": "ex:frank", "ex:name": "Frank" }, + { "@id": "ex:grace", "ex:name": "Grace" } + ] + } + }) + .to_string(), + ) + .send() + .await + .expect("transact request"); + assert_eq!(transact_resp.status(), reqwest::StatusCode::OK); + + let fluree = &state.fluree; + let reindex_result = fluree + .reindex("rawmode:test", ReindexOptions::default()) + .await + .expect("reindex should succeed"); + fluree + .refresh("rawmode:test", Default::default()) + .await + .expect("refresh after reindex should succeed"); + + // Resolve a real leaf CID from the index root via direct storage access. + let admin_storage = state + .fluree + .backend() + .admin_storage_cloned() + .expect("test backend has managed storage"); + let root_address = fluree_db_core::content_address( + "file", + ContentKind::IndexRoot, + "rawmode:test", + &reindex_result.root_id.digest_hex(), + ); + let direct_root_bytes = admin_storage + .read_bytes(&root_address) + .await + .expect("direct root read"); + let leaf_cid = extract_spot_leaf_cid(&direct_root_bytes); + let leaf_address = leaf_address_from_cid(&leaf_cid, "rawmode:test"); + let direct_leaf_bytes = admin_storage + .read_bytes(&leaf_address) + .await + .expect("direct leaf read"); + + let proxy_storage = ProxyStorage::new(server_url.clone(), token, ProxyReadMode::Raw); + + // Leaf: canonical FLI3 bytes, identical to origin storage, not FLKB. + let leaf_bytes = proxy_storage + .read_bytes(&leaf_address) + .await + .expect("raw leaf read should succeed"); + assert_eq!( + leaf_bytes, direct_leaf_bytes, + "raw-mode leaf bytes must be byte-identical to origin storage" + ); + assert!( + leaf_bytes.len() < 4 || &leaf_bytes[0..4] != FLKB_MAGIC, + "raw-mode leaf must not be FLKB transport format" + ); + + // Non-leaf (index root): identical round-trip through the objects endpoint. + let root_bytes = proxy_storage + .read_bytes(&root_address) + .await + .expect("raw root read should succeed"); + assert_eq!( + root_bytes, direct_root_bytes, + "raw-mode root bytes must be byte-identical to origin storage" + ); + + // Ranged reads: raw mode issues true HTTP Range requests (206) and the + // slice must match the equivalent slice of the direct bytes. + assert!(proxy_storage.supports_ranged_reads()); + let mid = (direct_leaf_bytes.len() / 2) as u64; + let ranged = proxy_storage + .read_byte_range(&leaf_address, 8..mid) + .await + .expect("ranged leaf read should succeed"); + assert_eq!( + ranged, + direct_leaf_bytes[8..mid as usize].to_vec(), + "ranged bytes must match the direct slice" + ); + // Range extending past the object is clamped. + let tail = proxy_storage + .read_byte_range(&leaf_address, mid..u64::MAX) + .await + .expect("tail range read should succeed"); + assert_eq!(tail, direct_leaf_bytes[mid as usize..].to_vec()); + // Range starting past the object length is empty (416 → empty slice). + let beyond = proxy_storage + .read_byte_range( + &leaf_address, + (direct_leaf_bytes.len() as u64 + 10)..u64::MAX, + ) + .await + .expect("out-of-range read should succeed as empty"); + assert!(beyond.is_empty()); + + // Out-of-scope token: scoped to a different ledger → NotFound (no + // existence leak). A token with no storage claims at all is rejected + // earlier by the extractor with 401, so scope-to-another-ledger is the + // case that exercises the per-ledger check. + let pubkey = signing_key.verifying_key().to_bytes(); + let pubkey_b64 = URL_SAFE_NO_PAD.encode(pubkey); + let header = serde_json::json!({ + "alg": "EdDSA", + "jwk": { "kty": "OKP", "crv": "Ed25519", "x": pubkey_b64 } + }); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let payload = serde_json::json!({ + "iss": did_from_pubkey(&pubkey), + "exp": now + 3600, + "iat": now, + "fluree.storage.all": false, + "fluree.storage.ledgers": ["other:ledger"] // NOT rawmode:test + }); + let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string().as_bytes()); + let payload_b64 = URL_SAFE_NO_PAD.encode(payload.to_string().as_bytes()); + let signing_input = format!("{header_b64}.{payload_b64}"); + let signature = signing_key.sign(signing_input.as_bytes()); + let sig_b64 = URL_SAFE_NO_PAD.encode(signature.to_bytes()); + let scoped_token = format!("{header_b64}.{payload_b64}.{sig_b64}"); + + let unauthorized_storage = + ProxyStorage::new(server_url.clone(), scoped_token, ProxyReadMode::Raw); + let err = unauthorized_storage + .read_bytes(&leaf_address) + .await + .expect_err("out-of-scope read must fail"); + assert!( + matches!(err, fluree_db_core::Error::NotFound(_)), + "out-of-scope raw read should surface NotFound, got: {err:?}" + ); + + server_handle.abort(); +} + +/// End-to-end peer test: a proxy-mode `Fluree` — memory storage backed by +/// `ProxyStorage` in Raw mode plus `ProxyNameService`, the exact wiring of +/// `build_proxy_fluree` — executes a query against an INDEXED ledger served +/// by the tx server over HTTP. +/// +/// This is the capability raw mode unlocks: the peer's binary index reader +/// consumes canonical FLI3 leaves/dicts fetched through `/storage/objects`. +/// (The filtered FLKB tier has no leaf decoder on the read path, so indexed +/// ledgers were previously unreadable through the proxy.) +/// +/// Multi-thread runtime is required: lazy dict materialization bridges +/// sync→async (block_on) during query execution, which would starve the +/// in-process axum server on a current-thread runtime. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_peer_proxy_fluree_queries_indexed_ledger() { + use fluree_db_api::{FlureeBuilder, NameServiceMode, ReindexOptions}; + use fluree_db_server::peer::{ProxyNameService, ProxyReadMode, ProxyStorage}; + use tokio::net::TcpListener; + + let (_tmp, state) = tx_server_state().await; + let app = build_router(state.clone()); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind to ephemeral port"); + let server_addr = listener.local_addr().expect("get local addr"); + let server_url = format!("http://{server_addr}"); + let server_handle = tokio::spawn(async move { + axum::serve(listener, app).await.expect("server run"); + }); + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + + let secret = [0u8; 32]; + let signing_key = SigningKey::from_bytes(&secret); + let token = create_storage_proxy_token_no_identity(&signing_key, true); + + // Create + populate + index a ledger on the tx server. + let client = reqwest::Client::new(); + let create_resp = client + .post(format!("{server_url}/v1/fluree/create")) + .header("content-type", "application/json") + .body(r#"{"ledger": "peerquery:test"}"#) + .send() + .await + .expect("create ledger request"); + assert_eq!(create_resp.status(), reqwest::StatusCode::CREATED); + + let transact_resp = client + .post(format!("{server_url}/v1/fluree/update")) + .header("content-type", "application/json") + .body( + serde_json::json!({ + "ledger": "peerquery:test", + "@context": { "ex": "http://example.org/ns/" }, + "insert": { + "@graph": [ + { "@id": "ex:hana", "ex:name": "Hana" }, + { "@id": "ex:ivan", "ex:name": "Ivan" } + ] + } + }) + .to_string(), + ) + .send() + .await + .expect("transact request"); + assert_eq!(transact_resp.status(), reqwest::StatusCode::OK); + + state + .fluree + .reindex("peerquery:test", ReindexOptions::default()) + .await + .expect("reindex should succeed"); + state + .fluree + .refresh("peerquery:test", Default::default()) + .await + .expect("refresh after reindex should succeed"); + + // Build a peer exactly like build_proxy_fluree does. + let peer_storage = ProxyStorage::new(server_url.clone(), token.clone(), ProxyReadMode::Raw); + let peer_ns = ProxyNameService::new(server_url.clone(), token); + let peer_fluree = FlureeBuilder::memory() + .build_with(peer_storage, NameServiceMode::ReadOnly(Arc::new(peer_ns))); + + // Query through the peer — all index reads go over HTTP. + let query = serde_json::json!({ + "@context": { "ex": "http://example.org/ns/" }, + "select": { "?s": ["ex:name"] }, + "where": { "@id": "?s", "ex:name": "?name" } + }); + let result = peer_fluree + .graph("peerquery:test") + .query() + .jsonld(&query) + .execute_formatted() + .await + .expect("peer query should succeed"); + + let rows = result.as_array().expect("query result should be an array"); + assert_eq!( + rows.len(), + 2, + "peer query should see both subjects: {result}" + ); + let names: Vec<&str> = rows + .iter() + .filter_map(|r| r.get("ex:name").and_then(|v| v.as_str())) + .collect(); + assert!( + names.contains(&"Hana") && names.contains(&"Ivan"), + "peer query should return indexed values, got: {result}" + ); + + server_handle.abort(); +} + +/// Remote mounts: a local Fluree (own ledgers, read-write) mounts a remote +/// Fluree's ledgers read-only under an alias prefix. +/// +/// Exercises the full composition: +/// - `CompositeNameService` routes `acme/…` lookups to the remote's +/// `ProxyNameService` and localizes records +/// - `StorageBackend::Routed` sends `acme/…` CAS reads through the mount's +/// `ProxyStorage` (raw, CID-verified, prefix-stripped) +/// - a mounted indexed ledger is queryable next to a local ledger, including +/// in one mixed dataset query +/// - writes to mounted aliases are rejected with a clear error +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_remote_mount_queries_and_write_rejection() { + use fluree_db_api::{FlureeBuilder, ReindexOptions, RemoteMountSpec}; + use fluree_db_server::peer::{ProxyNameService, ProxyReadMode, ProxyStorage}; + use tokio::net::TcpListener; + + // --- Remote origin: tx server with an indexed ledger --- + let (_tmp, state) = tx_server_state().await; + let app = build_router(state.clone()); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind to ephemeral port"); + let server_addr = listener.local_addr().expect("get local addr"); + let server_url = format!("http://{server_addr}"); + let server_handle = tokio::spawn(async move { + axum::serve(listener, app).await.expect("server run"); + }); + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + + let secret = [0u8; 32]; + let signing_key = SigningKey::from_bytes(&secret); + let token = create_storage_proxy_token_no_identity(&signing_key, true); + + let client = reqwest::Client::new(); + let create_resp = client + .post(format!("{server_url}/v1/fluree/create")) + .header("content-type", "application/json") + .body(r#"{"ledger": "inventory"}"#) + .send() + .await + .expect("create ledger request"); + assert_eq!(create_resp.status(), reqwest::StatusCode::CREATED); + let transact_resp = client + .post(format!("{server_url}/v1/fluree/update")) + .header("content-type", "application/json") + .body( + serde_json::json!({ + "ledger": "inventory", + "@context": { "ex": "http://example.org/ns/" }, + "insert": { "@id": "ex:widget", "ex:name": "Widget", "ex:sku": "W-1" } + }) + .to_string(), + ) + .send() + .await + .expect("transact request"); + assert_eq!(transact_resp.status(), reqwest::StatusCode::OK); + state + .fluree + .reindex("inventory:main", ReindexOptions::default()) + .await + .expect("reindex should succeed"); + state + .fluree + .refresh("inventory:main", Default::default()) + .await + .expect("refresh should succeed"); + + // --- Local Fluree with the remote mounted under "acme" --- + let mount_storage = ProxyStorage::new(server_url.clone(), token.clone(), ProxyReadMode::Raw) + .with_local_prefix("acme"); + let mount_ns = Arc::new(ProxyNameService::new(server_url.clone(), token)); + let local = FlureeBuilder::memory() + .with_remote_mount(RemoteMountSpec::new("acme", mount_ns, mount_storage)) + .build_memory(); + + // Local ledger with its own data. + local + .create_ledger("books") + .await + .expect("create local ledger"); + local + .graph("books:main") + .transact() + .insert(&serde_json::json!({ + "@context": { "ex": "http://example.org/ns/" }, + "@id": "ex:moby", "ex:name": "Moby Dick" + })) + .commit() + .await + .expect("local transact"); + + // Query the mounted (remote, indexed) ledger by its local alias. + let query = serde_json::json!({ + "@context": { "ex": "http://example.org/ns/" }, + "select": { "?s": ["ex:name"] }, + "where": { "@id": "?s", "ex:sku": "W-1" } + }); + let result = local + .graph("acme/inventory:main") + .query() + .jsonld(&query) + .execute_formatted() + .await + .expect("mounted query should succeed"); + let rows = result.as_array().expect("array result"); + assert_eq!(rows.len(), 1, "mounted query result: {result}"); + assert_eq!( + rows[0].get("ex:name").and_then(|v| v.as_str()), + Some("Widget"), + "mounted query should read remote index content: {result}" + ); + + // Mixed dataset: one query over the local ledger AND the mount. + let mixed = serde_json::json!({ + "@context": { "ex": "http://example.org/ns/" }, + "from": ["books:main", "acme/inventory:main"], + "select": ["?name"], + "where": { "@id": "?s", "ex:name": "?name" } + }); + let result = local + .query_from() + .jsonld(&mixed) + .execute_formatted() + .await + .expect("mixed dataset query should succeed"); + // Rows are single-var tuples: [["Moby Dick"], ["Widget"]] + let names: Vec<&str> = result + .as_array() + .expect("array result") + .iter() + .filter_map(|row| row.as_array()?.first()?.as_str()) + .collect(); + assert!( + names.contains(&"Widget") && names.contains(&"Moby Dick"), + "mixed dataset should span local + mounted ledgers, got: {result}" + ); + + // Writes to mounted aliases are rejected. The routed storage refuses + // first (ProxyStorage is read-only); the CompositeNameService guard + // backstops non-storage writes (create/branch/publish). + let err = local + .graph("acme/inventory:main") + .transact() + .insert(&serde_json::json!({ + "@context": { "ex": "http://example.org/ns/" }, + "@id": "ex:hack", "ex:name": "Nope" + })) + .commit() + .await + .expect_err("write to mount must fail"); + assert!( + err.to_string().contains("read-only"), + "unexpected write-rejection error: {err}" + ); + // Creating a local ledger that would shadow the mount is also rejected, + // at the nameservice layer. + let err = local + .create_ledger("acme/other") + .await + .expect_err("create under mount prefix must fail"); + assert!( + err.to_string().contains("read-only remote mount"), + "unexpected create-rejection error: {err}" + ); + + server_handle.abort(); +} + +/// Test per-ledger serving posture (`f:servingDefaults`) enforcement and +/// advertisement. +/// +/// - `f:serveBlocks false` → block/objects/commits endpoints return 404, +/// queries still served, NS record advertises `["query"]` +/// - `f:serveQuery false` (blocks restored) → query route returns 403 with a +/// stable message, blocks served again, NS record advertises `["blocks"]` +/// +/// Config changes are read novelty-inclusive, so no reindex is needed after +/// transacting the config graph. +#[tokio::test] +async fn test_serving_defaults_gate_and_advertisement() { + use fluree_db_api::ReindexOptions; + + let (_tmp, state) = tx_server_state().await; + let app = build_router(state.clone()); + + let secret = [0u8; 32]; + let signing_key = SigningKey::from_bytes(&secret); + let token = create_storage_proxy_token_no_identity(&signing_key, true); + + // Create + populate + index. + let create_body = serde_json::json!({ "ledger": "serving:test" }); + let resp = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/fluree/create") + .header("content-type", "application/json") + .body(Body::from(create_body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::CREATED); + + let data = serde_json::json!({ + "ledger": "serving:test", + "@context": { "ex": "http://example.org/ns/" }, + "insert": { "@id": "ex:judy", "ex:name": "Judy" } + }); + let resp = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/fluree/update") + .header("content-type", "application/json") + .body(Body::from(data.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let reindex_result = state + .fluree + .reindex("serving:test", ReindexOptions::default()) + .await + .expect("reindex should succeed"); + state + .fluree + .refresh("serving:test", Default::default()) + .await + .expect("refresh should succeed"); + let root_cid = reindex_result.root_id.to_string(); + + // Small helpers over the router. + let fetch_block_status = |app: axum::Router, token: String, cid: String| async move { + let body = serde_json::json!({ "cid": cid, "ledger": "serving:test" }); + let resp = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/fluree/storage/block") + .header("content-type", "application/json") + .header("Authorization", format!("Bearer {token}")) + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + resp.status() + }; + let query_status = |app: axum::Router| async move { + let body = serde_json::json!({ + "@context": { "ex": "http://example.org/ns/" }, + "from": "serving:test", + "select": ["?name"], + "where": { "@id": "?s", "ex:name": "?name" } + }); + let resp = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/fluree/query") + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + resp.status() + }; + let ns_serving = |app: axum::Router, token: String| async move { + let resp = app + .oneshot( + Request::builder() + .method("GET") + .uri("/v1/fluree/storage/ns/serving:test") + .header("Authorization", format!("Bearer {token}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let (status, json) = json_body(resp).await; + assert_eq!(status, StatusCode::OK, "NS record fetch failed: {json}"); + json.get("serving").cloned() + }; + + // Baseline: unconfigured ledger serves both tiers. + assert_eq!( + fetch_block_status(app.clone(), token.clone(), root_cid.clone()).await, + StatusCode::OK + ); + assert_eq!(query_status(app.clone()).await, StatusCode::OK); + assert_eq!( + ns_serving(app.clone(), token.clone()).await, + Some(serde_json::json!(["query", "blocks"])) + ); + + // Disable block serving via the config graph. + let cfg_trig = r" + @prefix f: . + @prefix rdf: . + + GRAPH { + rdf:type f:LedgerConfig . + f:servingDefaults . + f:serveBlocks false . + } + "; + state + .fluree + .graph("serving:test") + .transact() + .upsert_turtle(cfg_trig) + .commit() + .await + .expect("config transact should succeed"); + + // Blocks tier refused across all raw-content endpoints; query unaffected. + assert_eq!( + fetch_block_status(app.clone(), token.clone(), root_cid.clone()).await, + StatusCode::NOT_FOUND + ); + let objects_resp = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(format!( + "/v1/fluree/storage/objects/{root_cid}?ledger=serving:test" + )) + .header("Authorization", format!("Bearer {token}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(objects_resp.status(), StatusCode::NOT_FOUND); + let commits_resp = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri("/v1/fluree/commits/serving:test") + .header("Authorization", format!("Bearer {token}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(commits_resp.status(), StatusCode::NOT_FOUND); + assert_eq!(query_status(app.clone()).await, StatusCode::OK); + assert_eq!( + ns_serving(app.clone(), token.clone()).await, + Some(serde_json::json!(["query"])) + ); + + // Flip: blocks on, query off. + let cfg_trig = r" + @prefix f: . + + GRAPH { + f:serveBlocks true . + f:serveQuery false . + } + "; + state + .fluree + .graph("serving:test") + .transact() + .upsert_turtle(cfg_trig) + .commit() + .await + .expect("config transact should succeed"); + + assert_eq!( + fetch_block_status(app.clone(), token.clone(), root_cid.clone()).await, + StatusCode::OK + ); + assert_eq!(query_status(app.clone()).await, StatusCode::FORBIDDEN); + assert_eq!( + ns_serving(app.clone(), token.clone()).await, + Some(serde_json::json!(["blocks"])) + ); +} + +/// Discovery advertises the coarse server-level serving capabilities. +#[tokio::test] +async fn test_discovery_advertises_serving_capabilities() { + let (_tmp, state) = tx_server_state().await; + let app = build_router(state); + + let resp = app + .oneshot( + Request::builder() + .method("GET") + .uri("/.well-known/fluree.json") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let (status, json) = json_body(resp).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + json.get("serving"), + Some(&serde_json::json!({ "query": true, "blocks": true })), + "discovery should advertise serving capabilities: {json}" + ); +} + // ============================================================================= // Policy-Filtered FLKB Tests (PR6: Prove Filtered < Raw) // ============================================================================= diff --git a/fluree-vocab/src/lib.rs b/fluree-vocab/src/lib.rs index d6bc3a80c6..28f7bff5ad 100644 --- a/fluree-vocab/src/lib.rs +++ b/fluree-vocab/src/lib.rs @@ -2232,6 +2232,27 @@ pub mod config_iris { /// `f:target` — property IRI that a `FullTextProperty` entry applies to. pub const FULL_TEXT_TARGET: &str = "https://ns.flur.ee/db#target"; + + // ---- Serving defaults fields (ledger-scoped) ---- + + /// `f:servingDefaults` — serving-posture defaults on LedgerConfig. + /// Ledger-scoped: ignored on GraphConfig, not subject to override control. + /// Declares which serving tiers the ledger's origin server offers; gates + /// apply only on the origin (transaction-role) serving surface, never on + /// read-only peers/mounts querying their own replicated copy. + pub const SERVING_DEFAULTS: &str = "https://ns.flur.ee/db#servingDefaults"; + + /// `f:serveQuery` — boolean, origin serves query execution for this + /// ledger. Absent means allowed. + pub const SERVE_QUERY: &str = "https://ns.flur.ee/db#serveQuery"; + + /// `f:serveBlocks` — boolean, origin serves raw CAS blocks (storage + /// proxy / peer replication) for this ledger. Absent means allowed. + pub const SERVE_BLOCKS: &str = "https://ns.flur.ee/db#serveBlocks"; + + /// `f:publicVisibility` — boolean, ledger may be discovered and read + /// without authentication. Absent means false (token required). + pub const PUBLIC_VISIBILITY: &str = "https://ns.flur.ee/db#publicVisibility"; } // ============================================================================ From e036fa278a6ced65c5cc96be67980a209f962044 Mon Sep 17 00:00:00 2001 From: bplatz Date: Fri, 3 Jul 2026 13:15:04 -0400 Subject: [PATCH 2/9] feat(cli): peer-mode tracked ledgers with persistent CID cache - fluree track add --mode peer: queries execute locally against index blocks fetched on demand from the remote's raw storage tier (CID-verified), while writes and admin commands keep forwarding over HTTP (resolve_ledger_mode downgrades the peer target to Tracked). - Per-remote persistent artifact cache under the OS cache dir; entries are content-addressed and immutable so clearing is always safe. verify_freshness_on_cache_hit keeps heads current against the remote. - fluree remote ledgers : the remote's auth-filtered catalog with the serving tiers each ledger offers (query / blocks). - fluree cache status|clear for the peer cache. - track list shows the mode column; peer entries persist as mode = "peer" in [[tracked_ledgers]]. --- fluree-db-cli/src/cli.rs | 33 ++++ fluree-db-cli/src/commands/cache.rs | 98 ++++++++++ fluree-db-cli/src/commands/mod.rs | 1 + fluree-db-cli/src/commands/query.rs | 31 +++- fluree-db-cli/src/commands/remote.rs | 140 ++++++++++++++ fluree-db-cli/src/commands/track.rs | 18 +- fluree-db-cli/src/config.rs | 28 ++- fluree-db-cli/src/context.rs | 263 ++++++++++++++++++++++++--- fluree-db-cli/src/lib.rs | 2 + 9 files changed, 582 insertions(+), 32 deletions(-) create mode 100644 fluree-db-cli/src/commands/cache.rs diff --git a/fluree-db-cli/src/cli.rs b/fluree-db-cli/src/cli.rs index ed09716404..2e4702f140 100644 --- a/fluree-db-cli/src/cli.rs +++ b/fluree-db-cli/src/cli.rs @@ -1005,6 +1005,12 @@ pub enum Commands { action: TrackAction, }, + /// Inspect or clear the peer-mode index artifact cache + Cache { + #[command(subcommand)] + action: CacheAction, + }, + /// Build or update the binary index for a ledger /// /// Performs incremental indexing when possible (merges only new commits @@ -1771,6 +1777,12 @@ pub enum TrackAction { /// Alias on the remote (defaults to local alias) #[arg(long)] remote_alias: Option, + + /// Query execution mode: "proxy" (remote executes queries; default) + /// or "peer" (queries run locally over index blocks fetched from the + /// remote and cached by CID; requires a storage-scope token) + #[arg(long, value_parser = ["proxy", "peer"])] + mode: Option, }, /// Stop tracking a remote ledger @@ -1789,6 +1801,20 @@ pub enum TrackAction { }, } +#[derive(Subcommand)] +pub enum CacheAction { + /// Show peer-cache disk usage per remote + Status, + + /// Delete cached index artifacts (all remotes, or one with --remote). + /// Always safe: everything cached is content-addressed and re-fetchable. + Clear { + /// Only clear the cache for this remote + #[arg(long)] + remote: Option, + }, +} + #[derive(Subcommand)] pub enum ConfigAction { /// Get a configuration value @@ -2067,6 +2093,13 @@ pub enum RemoteAction { /// Remote name name: String, }, + + /// List the ledgers your token can access on a remote, with the + /// serving tiers each offers ("query", "blocks") + Ledgers { + /// Remote name; defaults to the only configured remote + name: Option, + }, } #[derive(Subcommand)] diff --git a/fluree-db-cli/src/commands/cache.rs b/fluree-db-cli/src/commands/cache.rs new file mode 100644 index 0000000000..7b26219999 --- /dev/null +++ b/fluree-db-cli/src/commands/cache.rs @@ -0,0 +1,98 @@ +//! Peer-cache inspection and cleanup: `fluree cache status|clear` +//! +//! The peer cache holds index artifacts fetched from remotes in peer mode +//! (see [`crate::context::peer_cache_root`]). Everything in it is +//! content-addressed and re-fetchable, so clearing is always safe. + +use crate::cli::CacheAction; +use crate::context::{peer_cache_dir, peer_cache_root}; +use crate::error::{CliError, CliResult}; +use colored::Colorize; +use std::fs; +use std::path::Path; + +pub fn run(action: CacheAction) -> CliResult<()> { + match action { + CacheAction::Status => run_status(), + CacheAction::Clear { remote } => run_clear(remote.as_deref()), + } +} + +fn run_status() -> CliResult<()> { + let root = peer_cache_root(); + if !root.exists() { + println!("Peer cache is empty ({}).", root.display()); + return Ok(()); + } + + let mut total: u64 = 0; + let mut rows: Vec<(String, u64)> = Vec::new(); + let entries = + fs::read_dir(&root).map_err(|e| CliError::Config(format!("read cache dir: {e}")))?; + for entry in entries.flatten() { + if entry.path().is_dir() { + let size = dir_size(&entry.path()); + total += size; + rows.push((entry.file_name().to_string_lossy().to_string(), size)); + } + } + rows.sort_by_key(|(_, size)| std::cmp::Reverse(*size)); + + if rows.is_empty() { + println!("Peer cache is empty ({}).", root.display()); + return Ok(()); + } + + println!("Peer cache: {}", root.display()); + for (remote, size) in &rows { + println!(" {:<24} {}", remote, human_bytes(*size)); + } + println!(" {:<24} {}", "total".bold(), human_bytes(total)); + Ok(()) +} + +fn run_clear(remote: Option<&str>) -> CliResult<()> { + let target = match remote { + Some(name) => peer_cache_dir(name), + None => peer_cache_root(), + }; + if !target.exists() { + println!("Nothing to clear ({}).", target.display()); + return Ok(()); + } + let freed = dir_size(&target); + fs::remove_dir_all(&target) + .map_err(|e| CliError::Config(format!("clear cache {}: {e}", target.display())))?; + println!("Cleared {} ({}).", target.display(), human_bytes(freed)); + Ok(()) +} + +fn dir_size(path: &Path) -> u64 { + let mut size = 0; + if let Ok(entries) = fs::read_dir(path) { + for entry in entries.flatten() { + let p = entry.path(); + if p.is_dir() { + size += dir_size(&p); + } else if let Ok(meta) = entry.metadata() { + size += meta.len(); + } + } + } + size +} + +fn human_bytes(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut value = bytes as f64; + let mut unit = 0; + while value >= 1024.0 && unit < UNITS.len() - 1 { + value /= 1024.0; + unit += 1; + } + if unit == 0 { + format!("{bytes} B") + } else { + format!("{value:.1} {}", UNITS[unit]) + } +} diff --git a/fluree-db-cli/src/commands/mod.rs b/fluree-db-cli/src/commands/mod.rs index b88c36bcb4..5b27c3ecb4 100644 --- a/fluree-db-cli/src/commands/mod.rs +++ b/fluree-db-cli/src/commands/mod.rs @@ -1,5 +1,6 @@ pub mod auth; pub mod branch; +pub mod cache; #[cfg(feature = "server")] pub mod cluster; pub mod completions; diff --git a/fluree-db-cli/src/commands/query.rs b/fluree-db-cli/src/commands/query.rs index 9b0040b802..43fc9258a1 100644 --- a/fluree-db-cli/src/commands/query.rs +++ b/fluree-db-cli/src/commands/query.rs @@ -377,8 +377,18 @@ pub async fn run( if is_cypher { // Cypher is local-only; it has no graph-source or connection form. + // A peer target provides a local (remote-backed) Fluree, so it runs + // like a local ledger under its remote alias. let mode = match target { context::QueryTarget::Ledger(mode) => mode, + context::QueryTarget::Peer { + fluree, + remote_alias, + .. + } => LedgerMode::Local { + fluree, + alias: remote_alias, + }, context::QueryTarget::GraphSource { .. } => { return Err(CliError::Usage( "Cypher queries are not supported on graph source targets".to_string(), @@ -437,6 +447,17 @@ pub async fn run( .await; } context::QueryTarget::Ledger(mode) => mode, + // Peer mode: local execution over the remote-backed Fluree. From here + // on it IS a local query — index blocks stream in over HTTP + // (CID-verified, disk-cached) as the engine touches them. + context::QueryTarget::Peer { + fluree, + remote_alias, + .. + } => LedgerMode::Local { + fluree, + alias: remote_alias, + }, }; match mode { @@ -1542,9 +1563,8 @@ fn target_endpoint_id(target: &context::QueryTarget) -> String { match target { context::QueryTarget::GraphSource { alias, .. } => alias.clone(), context::QueryTarget::Ledger(LedgerMode::Local { alias, .. }) => alias.clone(), - context::QueryTarget::Ledger(LedgerMode::Tracked { remote_alias, .. }) => { - remote_alias.clone() - } + context::QueryTarget::Ledger(LedgerMode::Tracked { remote_alias, .. }) + | context::QueryTarget::Peer { remote_alias, .. } => remote_alias.clone(), } } @@ -1752,7 +1772,10 @@ async fn run_connection_query( result } context::QueryTarget::Ledger(LedgerMode::Local { fluree, .. }) - | context::QueryTarget::GraphSource { fluree, .. } => { + | context::QueryTarget::GraphSource { fluree, .. } + // Peer: local federation over the remote-backed Fluree — FROM + // resolves remote aliases through its proxy nameservice. + | context::QueryTarget::Peer { fluree, .. } => { connection_query_local( &fluree, query_format, diff --git a/fluree-db-cli/src/commands/remote.rs b/fluree-db-cli/src/commands/remote.rs index 8a7d719e21..f3a3ef0984 100644 --- a/fluree-db-cli/src/commands/remote.rs +++ b/fluree-db-cli/src/commands/remote.rs @@ -20,9 +20,149 @@ pub async fn run(action: RemoteAction, dirs: &FlureeDir) -> CliResult<()> { RemoteAction::Remove { name } => run_remove(&store, &name).await, RemoteAction::List => run_list(&store).await, RemoteAction::Show { name } => run_show(&store, &name).await, + RemoteAction::Ledgers { name } => run_ledgers(&store, name.as_deref()).await, } } +/// List the ledgers the configured token can access on a remote, with the +/// serving tiers each offers. +/// +/// Uses the storage-scope endpoints: `GET /nameservice/snapshot` for the +/// auth-filtered catalog and `GET /storage/ns/{alias}` for the per-ledger +/// `serving` advertisement. +async fn run_ledgers(store: &TomlSyncConfigStore, name: Option<&str>) -> CliResult<()> { + let remotes = store + .list_remotes() + .await + .map_err(|e| CliError::Config(e.to_string()))?; + let remote = match name { + Some(n) => remotes + .iter() + .find(|r| r.name.as_str() == n) + .cloned() + .ok_or_else(|| CliError::NotFound(format!("remote '{n}' not found")))?, + None => match remotes.as_slice() { + [only] => only.clone(), + [] => { + return Err(CliError::Config( + "no remotes configured. Add one with `fluree remote add `".into(), + )) + } + _ => { + return Err(CliError::Input( + "multiple remotes configured; specify one: `fluree remote ledgers `" + .into(), + )) + } + }, + }; + + let base_url = match &remote.endpoint { + RemoteEndpoint::Http { base_url } => base_url.clone(), + _ => { + return Err(CliError::Config(format!( + "remote '{}' is not an HTTP remote", + remote.name.as_str() + ))); + } + }; + let token = remote.auth.token.clone().ok_or_else(|| { + CliError::Config(format!( + "remote '{}' has no token; `fluree remote ledgers` requires a bearer token \ + with storage scope.\n Log in with `fluree auth login {}`.", + remote.name.as_str(), + remote.name.as_str() + )) + })?; + + let client = reqwest::Client::new(); + let snapshot: serde_json::Value = client + .get(format!("{base_url}/nameservice/snapshot")) + .bearer_auth(&token) + .send() + .await + .map_err(|e| CliError::Remote(format!("snapshot request failed: {e}")))? + .error_for_status() + .map_err(|e| CliError::Remote(format!("snapshot request rejected: {e}")))? + .json() + .await + .map_err(|e| CliError::Remote(format!("snapshot response parse failed: {e}")))?; + + let ledgers = snapshot + .get("ledgers") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + + let mut table = Table::new(); + table.set_header(vec!["LEDGER", "COMMIT T", "INDEX T", "SERVING"]); + for record in &ledgers { + let Some(ledger_id) = record.get("ledger_id").and_then(|v| v.as_str()) else { + continue; + }; + // Per-ledger serving tiers come from the NS record lookup. + let serving = client + .get(format!( + "{base_url}/storage/ns/{}", + urlencoding::encode(ledger_id) + )) + .bearer_auth(&token) + .send() + .await + .ok() + .and_then(|r| r.error_for_status().ok()); + let serving = match serving { + Some(resp) => resp + .json::() + .await + .ok() + .and_then(|v| v.get("serving").cloned()) + .and_then(|s| { + s.as_array().map(|tiers| { + tiers + .iter() + .filter_map(|t| t.as_str()) + .collect::>() + .join("+") + }) + }) + .unwrap_or_else(|| "-".to_string()), + None => "-".to_string(), + }; + table.add_row(vec![ + Cell::new(ledger_id), + Cell::new( + record + .get("commit_t") + .and_then(serde_json::Value::as_i64) + .map_or_else(|| "-".to_string(), |t| t.to_string()), + ), + Cell::new( + record + .get("index_t") + .and_then(serde_json::Value::as_i64) + .map_or_else(|| "-".to_string(), |t| t.to_string()), + ), + Cell::new(serving), + ]); + } + + if ledgers.is_empty() { + println!( + "No ledgers visible on remote '{}' (token scope may be empty).", + remote.name.as_str() + ); + } else { + println!("{table}"); + println!( + "\n {} track one with `fluree track add --remote {} --mode peer`", + "hint:".dimmed(), + remote.name.as_str() + ); + } + Ok(()) +} + async fn run_add( store: &TomlSyncConfigStore, name: &str, diff --git a/fluree-db-cli/src/commands/track.rs b/fluree-db-cli/src/commands/track.rs index b6e1cd26e2..bce62a30d2 100644 --- a/fluree-db-cli/src/commands/track.rs +++ b/fluree-db-cli/src/commands/track.rs @@ -5,7 +5,7 @@ //! blocks are needed. use crate::cli::TrackAction; -use crate::config::{TomlSyncConfigStore, TrackedLedgerConfig}; +use crate::config::{TomlSyncConfigStore, TrackMode, TrackedLedgerConfig}; use crate::context::build_client_from_auth; use crate::error::{CliError, CliResult}; use colored::Colorize; @@ -22,12 +22,19 @@ pub async fn run(action: TrackAction, dirs: &FlureeDir) -> CliResult<()> { ledger, remote, remote_alias, + mode, } => { + let mode = match mode.as_deref() { + Some("peer") => TrackMode::Peer, + // clap's value_parser restricts to proxy|peer. + _ => TrackMode::Proxy, + }; run_add( &store, &ledger, remote.as_deref(), remote_alias.as_deref(), + mode, dirs, ) .await @@ -49,6 +56,7 @@ async fn run_add( ledger: &str, remote_name: Option<&str>, remote_alias: Option<&str>, + mode: TrackMode, dirs: &FlureeDir, ) -> CliResult<()> { // Resolve remote: explicit arg, or default if exactly one remote configured @@ -149,6 +157,7 @@ async fn run_add( local_alias: local_alias.clone(), remote: remote.name.as_str().to_string(), remote_alias: effective_remote_alias.to_string(), + mode, }; store.add_tracked(config)?; @@ -187,13 +196,18 @@ fn run_list(store: &TomlSyncConfigStore) -> CliResult<()> { } let mut table = Table::new(); - table.set_header(vec!["Local Alias", "Remote", "Remote Alias"]); + table.set_header(vec!["Local Alias", "Remote", "Remote Alias", "Mode"]); for t in tracked { + let mode = match t.mode { + TrackMode::Proxy => "proxy", + TrackMode::Peer => "peer", + }; table.add_row(vec![ Cell::new(&t.local_alias), Cell::new(&t.remote), Cell::new(&t.remote_alias), + Cell::new(mode), ]); } diff --git a/fluree-db-cli/src/config.rs b/fluree-db-cli/src/config.rs index 58e753bbd9..ba7787f5b1 100644 --- a/fluree-db-cli/src/config.rs +++ b/fluree-db-cli/src/config.rs @@ -438,14 +438,32 @@ use serde::{Deserialize, Serialize}; /// Configuration for a tracked (remote-only) ledger. /// -/// Tracked ledgers have no local data — all operations are proxied to the -/// remote server via HTTP. This is distinct from upstreams, which track -/// a local ledger's relationship to a remote for ref-level sync. +/// Tracked ledgers have no local data stored in `.fluree/storage` — this is +/// distinct from upstreams, which track a local ledger's relationship to a +/// remote for ref-level sync. The [`TrackMode`] selects how queries execute. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TrackedLedgerConfig { pub local_alias: String, pub remote: String, pub remote_alias: String, + /// How queries against this tracked ledger execute (default: proxy). + #[serde(default)] + pub mode: TrackMode, +} + +/// Query execution mode for a tracked ledger. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TrackMode { + /// Query-shipping: every query is an HTTP round-trip executed by the + /// remote server (its compute, row-level policy applied). + #[default] + Proxy, + /// Peer: queries execute locally against index blocks fetched on demand + /// from the remote's raw storage tier and cached by CID. Requires a + /// token with `fluree.storage.*` scope for the ledger. Writes still + /// forward to the remote over HTTP. + Peer, } /// TOML structure for sync configuration in config.toml @@ -709,6 +727,10 @@ impl TomlSyncConfigStore { "remote_alias", Value::from(tracked.remote_alias.as_str()).into(), ); + // Only write non-default modes to keep configs tidy. + if tracked.mode == TrackMode::Peer { + table.insert("mode", Value::from("peer").into()); + } tracked_aot.push(table); } diff --git a/fluree-db-cli/src/context.rs b/fluree-db-cli/src/context.rs index 658f931bea..3f3a160ab6 100644 --- a/fluree-db-cli/src/context.rs +++ b/fluree-db-cli/src/context.rs @@ -58,6 +58,26 @@ pub enum QueryTarget { /// A locally-registered graph source (Iceberg/R2RML). `alias` is normalized /// to `:main` for routing. GraphSource { fluree: Box, alias: String }, + /// A peer-tracked ledger: queries execute locally against index blocks + /// fetched on demand from the remote's raw storage tier (CID-verified, + /// cached on disk per remote). Only queries take this arm — + /// [`resolve_ledger_mode`] downgrades it to [`LedgerMode::Tracked`] so + /// every other command keeps forwarding over HTTP. + Peer { + /// Fluree wired to the remote via `ProxyStorage` (raw mode) + + /// `ProxyNameService`; queries run through the normal local path + /// against `remote_alias`. + fluree: Box, + /// HTTP client for the downgrade-to-Tracked path. + client: Box, + /// The alias on the remote server (the peer Fluree resolves this + /// directly — no local prefix). + remote_alias: String, + /// The local alias the user used. + local_alias: String, + /// The remote config name. + remote_name: String, + }, } /// Resolve a `fluree query` target, distinguishing native ledgers from @@ -142,17 +162,13 @@ pub async fn resolve_query_target( // Check tracked config let store = TomlSyncConfigStore::new(dirs.config_dir().to_path_buf()); if let Some(tracked) = store.get_tracked(ledger_part) { - return Ok(QueryTarget::Ledger( - build_tracked_mode(&store, &tracked, ledger_part).await?, - )); + return build_tracked_target(&store, &tracked, ledger_part).await; } // Also try the normalized ledger_id (user might have typed "mydb" but tracked as "mydb:main") if ledger_part != ledger_id { if let Some(tracked) = store.get_tracked(&ledger_id) { - return Ok(QueryTarget::Ledger( - build_tracked_mode(&store, &tracked, &ledger_id).await?, - )); + return build_tracked_target(&store, &tracked, &ledger_id).await; } } @@ -161,9 +177,7 @@ pub async fn resolve_query_target( if let Some(base) = ledger_part.split(':').next() { if base != ledger_part && base != ledger_id { if let Some(tracked) = store.get_tracked(base) { - return Ok(QueryTarget::Ledger( - build_tracked_mode(&store, &tracked, ledger_part).await?, - )); + return build_tracked_target(&store, &tracked, ledger_part).await; } } } @@ -195,6 +209,21 @@ pub async fn resolve_ledger_mode( ) -> CliResult { match resolve_query_target(explicit, dirs).await? { QueryTarget::Ledger(mode) => Ok(mode), + // Peer mode only changes where QUERIES execute; every other command + // (writes, admin, history) forwards over HTTP exactly like a + // proxy-tracked ledger. + QueryTarget::Peer { + client, + remote_alias, + local_alias, + remote_name, + .. + } => Ok(LedgerMode::Tracked { + client, + remote_alias, + local_alias, + remote_name, + }), QueryTarget::GraphSource { alias, .. } => Err(CliError::NotFound(format!( "'{alias}' is a registered graph source, not a ledger.\n \ Query it with `fluree query {alias}`, or inspect it with `fluree iceberg info {alias}`." @@ -253,12 +282,28 @@ async fn try_compound_remote_syntax( })) } -/// Build a `LedgerMode::Tracked` from a tracked config entry. -async fn build_tracked_mode( +/// Build a query target from a tracked config entry, honoring its +/// [`TrackMode`](crate::config::TrackMode) (proxy → HTTP query-shipping, +/// peer → local execution over remotely-fetched blocks). +async fn build_tracked_target( store: &TomlSyncConfigStore, tracked: &TrackedLedgerConfig, local_alias: &str, -) -> CliResult { +) -> CliResult { + if tracked.mode == crate::config::TrackMode::Peer { + return build_peer_target(store, tracked, local_alias).await; + } + Ok(QueryTarget::Ledger( + build_tracked_mode(store, tracked, local_alias).await?, + )) +} + +/// Resolve a tracked entry's remote to its HTTP endpoint + auth. +async fn tracked_remote_http( + store: &TomlSyncConfigStore, + tracked: &TrackedLedgerConfig, + local_alias: &str, +) -> CliResult<(String, fluree_db_nameservice_sync::RemoteAuth)> { let remote_name = RemoteName::new(&tracked.remote); let remote = store .get_remote(&remote_name) @@ -271,17 +316,23 @@ async fn build_tracked_mode( )) })?; - let base_url = match &remote.endpoint { - RemoteEndpoint::Http { base_url } => base_url.clone(), - _ => { - return Err(CliError::Config(format!( - "remote '{}' is not an HTTP remote; tracking requires HTTP", - tracked.remote - ))); - } - }; + match &remote.endpoint { + RemoteEndpoint::Http { base_url } => Ok((base_url.clone(), remote.auth)), + _ => Err(CliError::Config(format!( + "remote '{}' is not an HTTP remote; tracking requires HTTP", + tracked.remote + ))), + } +} - let client = build_client_from_auth(&base_url, &remote.auth); +/// Build a `LedgerMode::Tracked` from a tracked config entry. +async fn build_tracked_mode( + store: &TomlSyncConfigStore, + tracked: &TrackedLedgerConfig, + local_alias: &str, +) -> CliResult { + let (base_url, auth) = tracked_remote_http(store, tracked, local_alias).await?; + let client = build_client_from_auth(&base_url, &auth); Ok(LedgerMode::Tracked { client: Box::new(client), remote_alias: tracked.remote_alias.clone(), @@ -290,6 +341,72 @@ async fn build_tracked_mode( }) } +/// Build a [`QueryTarget::Peer`]: an embedded Fluree whose reads go to the +/// remote's raw storage tier (`ProxyStorage` in Raw mode, CID-verified) with +/// a persistent per-remote disk cache for index artifacts. +async fn build_peer_target( + store: &TomlSyncConfigStore, + tracked: &TrackedLedgerConfig, + local_alias: &str, +) -> CliResult { + use fluree_db_api::{LedgerManagerConfig, NameServiceMode}; + use fluree_db_nameservice_sync::{ProxyNameService, ProxyReadMode, ProxyStorage}; + + let (base_url, auth) = tracked_remote_http(store, tracked, local_alias).await?; + let token = auth.token.clone().ok_or_else(|| { + CliError::Config(format!( + "tracked ledger '{local_alias}' uses peer mode, which requires a bearer token \ + with storage scope for remote '{}'.\n \ + Log in with `fluree auth login {}` (or `--token`).", + tracked.remote, tracked.remote + )) + })?; + let client = build_client_from_auth(&base_url, &auth); + + // Remote base URLs are API bases (ending in `/fluree`), so use the + // api-base constructors rather than the server-root ones. + let storage = ProxyStorage::from_api_base(base_url.clone(), token.clone(), ProxyReadMode::Raw); + let ns = ProxyNameService::from_api_base(base_url, token); + + // Persistent, per-remote artifact cache. Everything cached is + // content-addressed and immutable, so entries never invalidate; the + // nameservice head lookup (verify_freshness_on_cache_hit) is the only + // per-query remote state. + let cache_config = LedgerManagerConfig { + cache_dir: peer_cache_dir(&tracked.remote), + verify_freshness_on_cache_hit: true, + ..Default::default() + }; + + let fluree = FlureeBuilder::memory() + .with_ledger_cache_config(cache_config) + .build_with(storage, NameServiceMode::ReadOnly(std::sync::Arc::new(ns))); + + Ok(QueryTarget::Peer { + fluree: Box::new(fluree), + client: Box::new(client), + remote_alias: tracked.remote_alias.clone(), + local_alias: local_alias.to_string(), + remote_name: tracked.remote.clone(), + }) +} + +/// Disk cache root for peer-mode index artifacts, per remote. +/// +/// Content is CID-addressed and immutable, so the cache is shared across +/// projects and safe to delete at any time (`fluree cache clear`). +pub fn peer_cache_dir(remote_name: &str) -> std::path::PathBuf { + peer_cache_root().join(remote_name) +} + +/// Root directory for all peer-mode caches. +pub fn peer_cache_root() -> std::path::PathBuf { + dirs::cache_dir() + .unwrap_or_else(std::env::temp_dir) + .join("fluree") + .join("peer-cache") +} + /// Build a `LedgerMode::Tracked` for a one-shot --remote flag. pub async fn build_remote_mode( remote_name_str: &str, @@ -676,7 +793,9 @@ mod tests { QueryTarget::GraphSource { alias, .. } => { assert_eq!(alias, "warehouse-orders:main"); } - QueryTarget::Ledger(_) => panic!("expected a GraphSource target, got a ledger"), + QueryTarget::Ledger(_) | QueryTarget::Peer { .. } => { + panic!("expected a GraphSource target, got a ledger") + } } } @@ -717,4 +836,102 @@ mod tests { Err(other) => panic!("expected NotFound, got {other:?}"), } } + + /// Seed a remote + tracked entry with the given mode and token. + async fn seed_tracked(dirs: &FlureeDir, mode: crate::config::TrackMode, token: Option<&str>) { + use fluree_db_nameservice_sync::{RemoteAuth, RemoteConfig, RemoteEndpoint}; + let store = TomlSyncConfigStore::new(dirs.config_dir().to_path_buf()); + store + .set_remote(&RemoteConfig { + name: RemoteName::new("origin"), + endpoint: RemoteEndpoint::Http { + base_url: "http://127.0.0.1:1/v1/fluree".to_string(), + }, + auth: RemoteAuth { + token: token.map(str::to_string), + ..Default::default() + }, + fetch_interval_secs: None, + }) + .await + .unwrap(); + store + .add_tracked(TrackedLedgerConfig { + local_alias: "inv:main".to_string(), + remote: "origin".to_string(), + remote_alias: "inventory:main".to_string(), + mode, + }) + .unwrap(); + } + + /// A peer-tracked ledger resolves to `QueryTarget::Peer` for queries + /// (local execution) while `resolve_ledger_mode` downgrades it to + /// `Tracked` so writes keep forwarding over HTTP. No network is + /// touched — building the peer target is offline. + #[tokio::test] + async fn peer_tracked_ledger_resolves_to_peer_target() { + let (_tmp, dirs) = temp_dirs(); + seed_tracked(&dirs, crate::config::TrackMode::Peer, Some("tok")).await; + + match resolve_query_target(Some("inv:main"), &dirs).await { + Ok(QueryTarget::Peer { + remote_alias, + local_alias, + remote_name, + .. + }) => { + assert_eq!(remote_alias, "inventory:main"); + assert_eq!(local_alias, "inv:main"); + assert_eq!(remote_name, "origin"); + } + Ok(_) => panic!("expected a Peer target"), + Err(e) => panic!("resolution failed: {e:?}"), + } + + match resolve_ledger_mode(Some("inv:main"), &dirs).await { + Ok(LedgerMode::Tracked { remote_alias, .. }) => { + assert_eq!(remote_alias, "inventory:main"); + } + Ok(LedgerMode::Local { .. }) => { + panic!("peer must downgrade to Tracked for non-query commands") + } + Err(e) => panic!("downgrade resolution failed: {e:?}"), + } + } + + /// Peer mode without a configured token fails with a pointer to + /// `fluree auth login`, not an opaque network error. + #[tokio::test] + async fn peer_tracked_ledger_without_token_errors_clearly() { + let (_tmp, dirs) = temp_dirs(); + seed_tracked(&dirs, crate::config::TrackMode::Peer, None).await; + + match resolve_query_target(Some("inv:main"), &dirs).await { + Err(CliError::Config(msg)) => { + assert!(msg.contains("peer mode"), "unexpected message: {msg}"); + assert!( + msg.contains("fluree auth login"), + "unexpected message: {msg}" + ); + } + Ok(_) => panic!("expected a config error"), + Err(other) => panic!("expected Config error, got {other:?}"), + } + } + + /// Default (proxy) tracked entries keep resolving to `Tracked`. + #[tokio::test] + async fn proxy_tracked_ledger_resolves_to_tracked() { + let (_tmp, dirs) = temp_dirs(); + seed_tracked(&dirs, crate::config::TrackMode::Proxy, Some("tok")).await; + + match resolve_query_target(Some("inv:main"), &dirs).await { + Ok(QueryTarget::Ledger(LedgerMode::Tracked { remote_alias, .. })) => { + assert_eq!(remote_alias, "inventory:main"); + } + Ok(_) => panic!("expected a Tracked target"), + Err(e) => panic!("resolution failed: {e:?}"), + } + } } diff --git a/fluree-db-cli/src/lib.rs b/fluree-db-cli/src/lib.rs index 81569b03ae..ca005d6127 100644 --- a/fluree-db-cli/src/lib.rs +++ b/fluree-db-cli/src/lib.rs @@ -602,6 +602,8 @@ pub async fn run(cli: Cli) -> error::CliResult<()> { commands::track::run(action, &fluree_dir).await } + Commands::Cache { action } => commands::cache::run(action), + Commands::Index { ledger } => { let fluree_dir = config::require_fluree_dir(config_path)?; commands::index::run_index(ledger.as_deref(), &fluree_dir).await From 777e6da785f2f0e703c359457ec37d1c62c7c886 Mon Sep 17 00:00:00 2001 From: bplatz Date: Fri, 3 Jul 2026 13:15:13 -0400 Subject: [PATCH 3/9] docs: remote mounts design and serving-tier documentation - New docs/design/remote-mounts.md: the serving-tier model (query / blocks / reserved filtered tier), per-caller resolution, mount architecture (CompositeNameService + StorageBackend::Routed + ProxyStorage modes), and the CID-verified cache-forever integrity semantics, with the fine-grained and vended-origin extension points. - setting-groups.md: f:servingDefaults as a ledger-scoped group. - query-peers.md: the two read tiers, corrected /storage/block leaf semantics, Range behavior, raw-tier access model. - auth-contract.md: discovery serving capability block. - CLI docs: track --mode peer, remote ledgers, new cache page. --- docs/SUMMARY.md | 2 + docs/cli/cache.md | 41 +++++ docs/cli/remote.md | 20 +++ docs/cli/track.md | 13 +- docs/design/README.md | 4 + docs/design/auth-contract.md | 5 +- docs/design/remote-mounts.md | 221 +++++++++++++++++++++++++++ docs/ledger-config/setting-groups.md | 23 +++ docs/operations/query-peers.md | 49 +++++- 9 files changed, 368 insertions(+), 10 deletions(-) create mode 100644 docs/cli/cache.md create mode 100644 docs/design/remote-mounts.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index de43277665..4762e54b72 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -37,6 +37,7 @@ - [publish](cli/publish.md) - [clone](cli/clone.md) - [track](cli/track.md) + - [cache](cli/cache.md) - [server](cli/server.md) - [memory](cli/memory.md) - [mcp](cli/mcp.md) @@ -82,6 +83,7 @@ - [Auth contract (CLI ↔ Server)](design/auth-contract.md) - [Nameservice schema v2](design/nameservice-schema-v2.md) - [Storage-agnostic commits and sync](design/storage-agnostic-commits-and-sync.md) + - [Remote mounts and serving tiers](design/remote-mounts.md) - [ContentId and ContentStore](design/content-id-and-contentstore.md) - [Index format](design/index-format.md) - [Edge annotations (storage internals)](design/edge-annotations.md) diff --git a/docs/cli/cache.md b/docs/cli/cache.md new file mode 100644 index 0000000000..863150abc5 --- /dev/null +++ b/docs/cli/cache.md @@ -0,0 +1,41 @@ +# fluree cache + +Inspect or clear the peer-mode index artifact cache. + +When a tracked ledger uses `--mode peer` (see [track](track.md)), queries +execute locally against index blocks fetched from the remote's raw storage +tier. Fetched artifacts are cached on disk, keyed by remote, under the OS +cache directory (`/fluree/peer-cache//`). Everything in +the cache is content-addressed and immutable — entries never go stale and +clearing is always safe (artifacts re-fetch on demand). + +## Subcommands + +### fluree cache status + +Show per-remote disk usage. + +```bash +fluree cache status +``` + +``` +Peer cache: /Users/me/Library/Caches/fluree/peer-cache + origin 412.3 MiB + analytics 88.1 MiB + total 500.4 MiB +``` + +### fluree cache clear + +Delete cached artifacts — everything, or one remote's with `--remote`. + +```bash +fluree cache clear # all remotes +fluree cache clear --remote origin # one remote +``` + +## See Also + +- [track](track.md) - Track a remote ledger (peer mode) +- [remote](remote.md) - Manage remotes and list accessible ledgers diff --git a/docs/cli/remote.md b/docs/cli/remote.md index 9b8d46eea2..73f4c1be7e 100644 --- a/docs/cli/remote.md +++ b/docs/cli/remote.md @@ -124,6 +124,26 @@ Remote: Auth: token configured ``` +### fluree remote ledgers + +List the ledgers your token can access on a remote, with the serving tiers +each offers (`query` = the remote executes queries; `blocks` = raw index +content is served for peer-mode local execution). Requires a bearer token +with storage scope; the listing is the remote's auth-filtered catalog. + +```bash +fluree remote ledgers [NAME] # NAME defaults to the only configured remote +``` + +``` +LEDGER COMMIT T INDEX T SERVING +inventory:main 42 40 query+blocks +orders:main 17 17 query +``` + +Track one for peer-mode local querying with +`fluree track add --remote --mode peer` (see [track](track.md)). + ## See Also - [upstream](upstream.md) - Configure upstream tracking diff --git a/docs/cli/track.md b/docs/cli/track.md index 336e23a140..110bc56c07 100644 --- a/docs/cli/track.md +++ b/docs/cli/track.md @@ -1,6 +1,11 @@ # fluree track -Track a remote ledger without storing local data. Tracked ledgers route reads and writes to the configured remote server while keeping a lightweight record locally so you can use short aliases and the active-ledger shortcut. +Track a remote ledger without storing local data. Tracked ledgers keep a lightweight record locally so you can use short aliases and the active-ledger shortcut. Two query modes are available: + +- **proxy** (default): reads and writes route to the remote server over HTTP — the remote executes your queries (its compute, row-level policy applied). +- **peer**: queries execute **locally** against index blocks fetched on demand from the remote's raw storage tier, CID-verified and cached on disk per remote (see `fluree cache`). Writes still forward to the remote over HTTP. Requires a bearer token with `fluree.storage.*` scope for the ledger — the remote serves its full contents, so this mode is only offered for ledgers you may read in full (see [Remote mounts and serving tiers](../design/remote-mounts.md)). + +Use `fluree remote ledgers ` to see which ledgers your token can access and which serving tiers (`query`, `blocks`) each offers. ## Usage @@ -17,7 +22,7 @@ Start tracking a remote ledger under a local alias. **Usage:** ```bash -fluree track add [--remote ] [--remote-alias ] +fluree track add [--remote ] [--remote-alias ] [--mode ] ``` **Arguments:** @@ -32,6 +37,7 @@ fluree track add [--remote ] [--remote-alias ] |--------|-------------| | `--remote ` | Remote name (e.g., `origin`). Defaults to the only configured remote if unambiguous. | | `--remote-alias ` | Alias on the remote (defaults to the local alias) | +| `--mode ` | Query execution mode (default `proxy`; see above) | **Examples:** @@ -41,6 +47,9 @@ fluree track add production --remote origin # Use a different local alias fluree track add prod --remote origin --remote-alias production + +# Peer mode: local query execution over remotely-served index blocks +fluree track add analytics --remote origin --mode peer ``` ### fluree track remove diff --git a/docs/design/README.md b/docs/design/README.md index 723bb507a0..c989b39ede 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -20,6 +20,10 @@ Design of the nameservice schema: ledger records, graph source records, configur How ContentId (CIDv1) values decouple the commit chain from storage backends, enabling replication across filesystem, S3, and IPFS. Includes the pack protocol wire format for efficient bulk transfer. +### [Remote mounts and serving tiers](remote-mounts.md) + +How a Fluree server exposes ledgers to other instances (query tier vs raw-block tier, per-ledger `f:servingDefaults` posture, token scoping) and how a consumer mounts a remote's ledgers read-only under an alias prefix: `CompositeNameService`, `StorageBackend::Routed`, `ProxyStorage` raw/filtered modes, and the CID-verified cache-forever integrity model. + ### [ContentId and ContentStore](content-id-and-contentstore.md) The content-addressed identity layer: `ContentId` type, `ContentStore` trait, multicodec content kinds, and the bridge between CID-based identity and storage-backend addressing. diff --git a/docs/design/auth-contract.md b/docs/design/auth-contract.md index bfe5ce865d..865981d183 100644 --- a/docs/design/auth-contract.md +++ b/docs/design/auth-contract.md @@ -51,10 +51,13 @@ The CLI fetches this endpoint when a remote is added (`fluree remote add`) to au "exchange_url": "https://data.example.com/v1/fluree/auth/exchange", "scopes": ["openid", "profile"], "redirect_port": 8400 - } + }, + "serving": { "query": true, "blocks": true } } ``` +The optional `serving` object advertises the server-wide serving capabilities: `query` (the server executes queries) and `blocks` (the storage proxy serves raw replication content, enabling peer/local-compute mode). Per-ledger posture may further restrict either tier — the authoritative per-ledger view is the `serving` array on `GET /v1/fluree/storage/ns/{ledger-id}` responses (see [Query peers](../operations/query-peers.md)). + ### `api_base_url` `api_base_url` tells the CLI where the Fluree HTTP API is mounted. diff --git a/docs/design/remote-mounts.md b/docs/design/remote-mounts.md new file mode 100644 index 0000000000..377a2fe559 --- /dev/null +++ b/docs/design/remote-mounts.md @@ -0,0 +1,221 @@ +# Remote mounts and serving tiers + +This document describes how one Fluree server exposes ledgers to other Fluree +instances (servers or CLI clients), and how a consuming instance mounts a +remote's ledgers as read-only, locally-queryable data sources. It covers the +serving-tier model, per-ledger serving posture, the mount architecture +(composite nameservice + routed storage), and the integrity/caching +semantics that make remotely-fetched content safe to cache indefinitely. + +Related documents: [Query peers and replication](../operations/query-peers.md) +(operator guide), [Auth contract](auth-contract.md) (token/claim wire +contract), [Setting groups](../ledger-config/setting-groups.md) +(`f:servingDefaults`), [Storage traits](storage-traits.md). + +## The model in one paragraph + +A serving Fluree offers each ledger through up to two tiers: **query** +(the server executes queries, with row-level policy applied — its compute) +and **blocks** (the server hands out canonical content-addressed bytes — +index leaves, dictionaries, commits — and the consumer executes queries +locally over them: the consumer's compute, like an Iceberg catalog serving +table files). Access to the blocks tier is **all-or-nothing per (token, +ledger)**: a principal either may read the ledger's full contents or gets +nothing. Fine-grained (row-level) access is served exclusively through the +query tier. A consumer *mounts* a remote under an alias prefix; the remote's +ledgers then behave like local read-only ledgers — full triple semantics, +time-travel, dataset mixing — with all bytes fetched over HTTP on demand and +cached by CID. + +## Serving tiers + +| Tier | Endpoint(s) | Whose compute | Access granularity | Payload integrity | +|---|---|---|---|---| +| `query` | `/query`, `/query/*ledger`, SPARQL | Server's | Row-level (identity policy) | N/A (results, not content) | +| `blocks` | `GET /storage/objects/{cid}`, `POST /pack`, `GET /commits` | Consumer's | Full ledger or nothing | CID-verified, canonical | +| filtered blocks (reserved) | `POST /storage/block` (FLKB leaf payloads) | Mixed | Row-level | Not CID-verifiable | + +The third row exists in the wire protocol (the storage-block endpoint always +policy-filters leaf payloads and never returns raw FLI3) but has no +production consumer: nothing on the read path decodes FLKB leaves. It is the +reserved transport for future fine-grained peer access; see +[Fine-grained future](#fine-grained-future). + +Writes are tier-independent: they always ship to the origin's transaction +data plane (`/transact`, `/insert`, …) because commits are ordered by the +ledger's write authority. A consumer with block access and write scope reads +locally and writes remotely. + +### Per-(caller, ledger) resolution + +The effective tiers for a request are the intersection of three layers: + +1. **Server capability** — queries are always served; the blocks tier + requires the storage proxy to be enabled (`StorageProxyConfig`). +2. **Ledger posture** — the `f:servingDefaults` setting group in the + ledger's config graph: `f:serveQuery`, `f:serveBlocks` (absent = allowed), + `f:publicVisibility` (absent = token required). See + [setting groups](../ledger-config/setting-groups.md). +3. **Token claims** — `fluree.ledger.read.*` for the query tier; + `fluree.storage.all` / `fluree.storage.ledgers` for the blocks tier + (full-access replication scope; see [auth contract](auth-contract.md)). + +Enforcement points: the query gate runs in the query-route ledger load +(403 with a stable message); the blocks gate runs on `/storage/block`, +`/storage/objects`, `/commits`, and `/pack` (404, no existence leak). + +**Serving posture binds only the origin.** `f:servingDefaults` lives in the +ledger's config graph, which replicates with the ledger — but the gates are +enforced only on transaction-role servers. A read-only peer or a consumer +that mounted the blocks always queries its own copy freely: restricting what +a holder of the full bytes does locally is not enforceable, and blocking it +would defeat the purpose of block serving. "Query serving off" therefore +means "the *origin* won't spend query compute", not "this data may not be +queried." + +### Advertisement + +- `/.well-known/fluree.json` carries a coarse, unauthenticated + `"serving": {"query": bool, "blocks": bool}` server capability block. +- `GET /storage/ns/{alias}` (authenticated) annotates each nameservice + record with the computed per-ledger tiers: `"serving": ["query","blocks"]`. + Because consumers fetch this record for head freshness anyway, + mode negotiation costs no extra round-trips. +- `GET /nameservice/snapshot` lists the records the token may see (scope + filtering, 404 anti-leak) — the auth-filtered catalog. + +## Mount architecture + +A mount makes remote ledgers appear under a local alias prefix: mount +`acme` exposes the remote's `inventory:main` as `acme/inventory:main`. +Two seams carry the whole composition; nothing else in the engine knows +mounts exist. + +### Nameservice: `CompositeNameService` + +`fluree-db-nameservice/src/mount.rs`. A composite over the local +read-write nameservice and N read-only mounts (`RemoteMount` = prefix + +`Arc`): + +- **Reads** route by prefix: `acme/inventory:main` → strip → remote lookup + `inventory:main` → the returned record is *localized* (its `ledger_id` and + `name` re-prefixed) so every downstream consumer — ledger cache keys, + content-store namespacing, branched-store ancestry walks — operates on the + local alias. +- **Writes** to mounted aliases fail with a "read-only remote mount" error; + writes to local aliases delegate to the local publisher. This includes + `init`, so a local ledger cannot be created that shadows a mount. +- The composite implements the full `NameServicePublisher` surface, giving + per-alias write capability on top of `NameServiceMode`'s instance-level + read-write/read-only split. + +### Storage: `StorageBackend::Routed` + +`fluree-db-core/src/storage.rs`. `StorageBackend::content_store(namespace)` +is the single point where a ledger's namespace binds to a store. The +`Routed` variant holds a default backend plus `(prefix, backend)` mounts and +selects by namespace prefix at exactly that point — so `LedgerState::load`, +`BranchedContentStore` ancestry, and default-context reads all route with no +changes. Admin operations (delete, list) apply only to the default backend. + +### Transport: `ProxyStorage` / `ProxyNameService` + +`fluree-db-nameservice-sync` (the HTTP client crate; the server re-exports +them under `fluree_db_server::peer` for the whole-server peer mode). + +`ProxyStorage` implements the `Storage` traits over HTTP with an explicit +read mode: + +- **`ProxyReadMode::Raw`** — fetches canonical bytes via + `GET /storage/objects/{cid}` and verifies every full payload against its + CID client-side. Supports true HTTP Range reads (`Range: bytes=…` → 206), + where the server verifies the full object before slicing. This is the mode + peers and mounts use; it is what makes binary-indexed ledgers readable + remotely (FLI3 leaves arrive canonical). +- **`ProxyReadMode::Filtered`** — the FLKB negotiation against + `POST /storage/block`; reserved for fine-grained access. + +For mounts, `ProxyStorage::with_local_prefix` strips the mount prefix from +locally-derived aliases before requests go to the remote (the inverse of the +composite's record localization). Dict blobs need one special case: their +`@shared` addresses carry only the ledger *name*, so the client derives a +default-branch alias and the server branch-resolves dict-blob requests to +any live branch of the name. + +### Assembly + +`FlureeBuilder::with_remote_mount(RemoteMountSpec)` (fluree-db-api). At +build time the backend is wrapped in `Routed` and the nameservice in a +composite. The spec takes transport-agnostic parts +(`Arc` + `impl Storage`), keeping HTTP out of the API +crate: + +```rust +let storage = ProxyStorage::new(url, token, ProxyReadMode::Raw) + .with_local_prefix("acme"); +let ns = Arc::new(ProxyNameService::new(url, token)); +let fluree = FlureeBuilder::memory() + .with_remote_mount(RemoteMountSpec::new("acme", ns, storage)) + .build_memory(); +// fluree.graph("acme/inventory:main").query()… — local compute, remote bytes +``` + +Mounted and local ledgers mix freely in one dataset query +(`"from": ["books:main", "acme/inventory:main"]`). + +The whole-server **peer mode** (`--storage-access-mode proxy`) is the +degenerate case of the same components: one upstream, no prefix, read-only +nameservice mode. + +## Integrity and caching semantics + +Everything the blocks tier serves is canonical content-addressed data, which +gives the consumer-side cache its key property: **a cached CID is valid +forever; eviction is always safe; no invalidation exists**. Only the +nameservice head lookup is per-query state. + +- Raw payloads are verified against their CID on the client (and on the + server before serving). Ranged responses are verified server-side against + the full object. +- Filtered (FLKB) payloads are *not* canonical: they are a function of + (CID, identity, policy-at-fetch-time) and must never be written into a + CAS store or a shared cache under the object's CID — a policy revocation + must not be servable from a stale cached view. Any future cache for them + needs identity- and policy-epoch-scoped keys. +- Origin-side garbage collection composes naturally: a consumer's cache miss + on a collected CID is a 404, bounding time-travel by the origin's + retention. + +## Relationship to adjacent mechanisms + +- **SPARQL `SERVICE fluree:remote:…`** — query shipping (origin's compute); + complementary to mounts (consumer's compute). A client negotiates between + them from the advertised serving tiers. +- **`fetch`/`pull`/`clone` (nameservice-sync)** — eager full replication + into local storage. A mount is the lazy, demand-driven counterpart over + the same raw content; a warm mount cache is effectively a partial clone. +- **Graph sources (Iceberg/R2RML)** — for non-RDF backends with no Fluree + index. Remote Fluree ledgers deliberately do *not* use the graph-source + machinery: riding the native ledger path preserves time-travel, policy, + and dataset semantics. + +## Fine-grained future + +The reserved extension points for row-level peer access, kept so it can be +added without reworking v1: + +- The `POST /storage/block` FLKB tier already produces policy-filtered leaf + payloads per identity; a read-path FLKB leaf decoder is the missing + consumer. +- Serving advertisement uses distinct values — a future filtered tier + advertises as `"blocks:filtered"`, never as `"blocks"`, so existing + clients cannot misread filtered payloads as canonical. +- Cache writes are discriminated by CID-verifiability; filtered payloads + arrive unverifiable and take identity/epoch-scoped cache keys. +- A partial-access block claim would be a new claim name, not a + reinterpretation of `fluree.storage.*` (which remains full-access). + +Similarly reserved: `f:publicVisibility` for an anonymous (public dataset) +tier, and origin/vended-credential handoff (`LedgerConfig.origins` already +models prioritized external origins — S3/IPFS/CDN — so a full-access +consumer could bypass the proxy entirely). diff --git a/docs/ledger-config/setting-groups.md b/docs/ledger-config/setting-groups.md index bb51ca6cd8..8baf1d60f5 100644 --- a/docs/ledger-config/setting-groups.md +++ b/docs/ledger-config/setting-groups.md @@ -363,6 +363,29 @@ Some settings are structurally tied to the ledger as a whole and are **not meani Override control does not apply to ledger-scoped settings — they are changed only by writing to the config graph. +### `f:servingDefaults` — serving posture + +Declares which serving tiers the ledger's **origin server** offers to callers. Fields (all optional; absent means allowed): + +| Field | Type | Meaning | +|---|---|---| +| `f:serveQuery` | boolean | Origin executes queries for this ledger (`false` → query endpoints return 403 with a stable message) | +| `f:serveBlocks` | boolean | Origin serves raw replication content — storage-proxy blocks/objects, commit blobs, pack streams (`false` → those endpoints return 404) | +| `f:publicVisibility` | boolean | Ledger is discoverable/readable without a token (default `false`; reserved for the anonymous access tier) | + +```trig +GRAPH { + a f:LedgerConfig ; + f:servingDefaults . + f:serveQuery false ; + f:serveBlocks true . +} +``` + +The example above is the "bring your own compute" posture: consumers fetch index blocks and execute queries client-side (peer mode); the origin refuses to spend query compute. + +Serving gates bind **only the origin's serving surface** (transaction-role servers). A read-only peer or a consumer that mounts the ledger's blocks always queries its own copy freely — the posture travels with the config graph but is deliberately not enforced on replicas, since restricting what a holder of the full blocks does locally is not enforceable anyway. Peers negotiate the mode via the `serving` field on nameservice record responses (see [Query peers](../operations/query-peers.md)). + > **Note:** `f:authzSource` (an identity/relationship graph used by policy evaluation) is planned as a ledger-scoped setting but is not yet implemented. When available, it will let the config graph specify which graph contains identity data (e.g., DID→role mappings) for policy resolution. --- diff --git a/docs/operations/query-peers.md b/docs/operations/query-peers.md index 20e2bf758c..4fa20a33f0 100644 --- a/docs/operations/query-peers.md +++ b/docs/operations/query-peers.md @@ -68,18 +68,30 @@ Peer servers support two storage access modes: - Requires `--storage-path`. - **Proxy storage** (`--storage-access-mode proxy`) - The peer does **not** need direct storage credentials. - - The peer proxies all storage reads through the transaction server’s `/v1/fluree/storage/*` endpoints. - - Requires `--tx-server-url` and a **storage proxy token** via `--storage-proxy-token` or `--storage-proxy-token-file`. + - The peer fetches all storage reads through the transaction server’s `/v1/fluree/storage/objects/{cid}` endpoint (raw CAS bytes, verified against the CID client-side), including index leaves and dictionary blobs — so queries execute locally on the peer against remotely served index content. + - Requires `--tx-server-url` and a **storage proxy token** via `--storage-proxy-token` or `--storage-proxy-token-file`. The token's ledger scope is full-access per ledger (see the storage proxy section below). - `--storage-path` is ignored in this mode. ## Storage proxy endpoints (transaction server): `/v1/fluree/storage/*` Storage proxy endpoints allow a peer to read storage **through** the transaction server, rather than holding storage credentials directly. This is intended for environments where storage is private and peers cannot access it. -Storage proxy supports two kinds of reads: +Storage proxy supports two read tiers: -- **Raw bytes** reads (`Accept: application/octet-stream`) for any block type (commit blobs, branch nodes, leaf nodes). -- **Policy-filtered leaf flakes** reads (`Accept: application/x-fluree-flakes`) for ledger **leaf** nodes only. +- **Raw CAS object reads** (`GET /v1/fluree/storage/objects/{cid}`): canonical + content-addressed bytes for any replication-relevant kind, including raw + index leaves and dictionary blobs. Bytes are integrity-verified against the + CID by the server before serving, and clients verify them again on receipt. + This tier serves **full ledger content without policy filtering** — the + bearer token's ledger scope is the access decision. +- **Policy-mediated block reads** (`POST /v1/fluree/storage/block`): leaf + blocks are always decoded and policy-filtered before transport (FLKB + format); raw FLI3 leaf bytes are never returned on this endpoint. Non-leaf + blocks are returned as raw bytes. + +Peers in proxy storage mode read through the **raw object tier** (with +client-side CID verification). The policy-mediated tier is the transport for +future fine-grained (row-filtered) peer access. ### Enablement @@ -107,11 +119,34 @@ Unauthorized requests return **404** (no existence leak). Fetch a nameservice record for a ledger ID. Requires storage proxy authorization for that ledger. +The response includes a `serving` array (`"query"`, `"blocks"`) advertising the tiers this server offers for the ledger, computed from the ledger's `f:servingDefaults` setting group (see [setting groups](../ledger-config/setting-groups.md)). Clients use it to negotiate between query-shipping and peer (local compute) modes. A ledger with `f:serveBlocks false` returns 404 from all raw-content endpoints (`/storage/block`, `/storage/objects`, `/commits`, `/pack`); one with `f:serveQuery false` returns 403 from query endpoints on the origin. The server-wide coarse view is advertised unauthenticated in `/.well-known/fluree.json` under `serving`. + +#### `GET /v1/fluree/storage/objects/{cid}?ledger={ledger-id}` + +Fetch a CAS object by **CID** as canonical raw bytes (`application/octet-stream`). +Serves all replication-relevant kinds: commits, txns, ledger config, index +roots, branches, **raw FLI3 leaves**, and dictionary blobs; only internal GC +records are refused. The server verifies the bytes against the CID before +responding (a mismatch is treated as storage corruption), and clients verify +again on receipt — the payload is canonical, so it is safe to cache +indefinitely under its CID. + +Because raw index content bypasses policy filtering, this endpoint's access +model is **all-or-nothing per ledger**: the token's `fluree.storage.*` scope +must cover the requested ledger, and such tokens must only be issued to +principals entitled to the ledger's full contents. + +Single-range `Range: bytes=start-end` requests are honored with a 206 +Partial Content response (`Content-Range` included). The server verifies the +**full** object against its CID before slicing, so partial responses carry +the same corruption guarantee; peers use this for leaflet-granular index +reads instead of pulling whole objects. + #### `POST /v1/fluree/storage/block` -Fetch a block/blob by **CID**. The request includes the **ledger ID** so the server can authorize the request and derive the physical storage address internally. Currently supports: +Fetch a block/blob by **CID** with policy mediation. The request includes the **ledger ID** so the server can authorize the request and derive the physical storage address internally. Currently supports: -- `Accept: application/octet-stream` (raw bytes; always available) +- `Accept: application/octet-stream` (raw bytes for **non-leaf** blocks; leaf blocks still return FLKB — raw FLI3 leaves are never served on this endpoint) - `Accept: application/x-fluree-flakes` (binary “FLKB” transport of policy-filtered **leaf** flakes only) - `Accept: application/x-fluree-flakes+json` (debug-only JSON flake transport; leaf flakes only) From cfd1f90521b9ff4ab11d39f6ad5cf9c49997e6a6 Mon Sep 17 00:00:00 2001 From: bplatz Date: Fri, 3 Jul 2026 13:15:13 -0400 Subject: [PATCH 4/9] memory --- .fluree-memory/repo.ttl | 90 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/.fluree-memory/repo.ttl b/.fluree-memory/repo.ttl index 89e2da3895..b29fd8b8fc 100644 --- a/.fluree-memory/repo.ttl +++ b/.fluree-memory/repo.ttl @@ -1461,6 +1461,22 @@ mem:constraint-01kwjtjew74a727brs0q3jaf7e a mem:Constraint ; mem:branch "feature/rdfs-enforcement-entailment" ; mem:createdAt "2026-07-03T01:47:55.655870+00:00"^^xsd:dateTime . +mem:decision-01kwm1agwk619zyf6rsxrjj7ap a mem:Decision ; + mem:content "Remote-mount / query-peer feature v1 scope: ledger access over the block-serving (peer) path is all-or-nothing per (token, ledger) — full-access raw CAS bytes (CID-verifiable, globally cacheable) or nothing; plus a public/anonymous tier. No policy-filtered (FLKB) leaf serving to peers in v1 — fine-grained permissions require query-shipping (server executes with row-level policy). Requires adding a full-access raw mode to the storage proxy (EnforcementMode::TrustedInternal gated on token scope; today PolicyEnforced/FLKB is the only proxy mode). v2 hooks: FLKB negotiation stays in ProxyStorage, serving advertisement uses distinct \"blocks\" vs future \"blocks:filtered\", cache writes carry a CID-verified flag." ; + mem:tag "caching" ; + mem:tag "feature-design" ; + mem:tag "peer-mode" ; + mem:tag "policy" ; + mem:tag "remote-mounts" ; + mem:tag "storage-proxy" ; + mem:scope mem:repo ; + mem:artifactRef "fluree-db-api/src/block_fetch.rs" ; + mem:artifactRef "fluree-db-server/src/peer/proxy_storage.rs" ; + mem:artifactRef "fluree-db-server/src/routes/storage_proxy.rs" ; + mem:branch "feature/rdfs-enforcement-entailment" ; + mem:createdAt "2026-07-03T13:05:10.035196+00:00"^^xsd:dateTime ; + mem:rationale "Avoids the two hardest v1 problems: filtered leaves are not immutable (policy changes make cached filtered views a revocation leak) and non-byte-identical payloads break CAS integrity checks. All-or-nothing keeps every peer-fetched block canonical and cache-forever." . + mem:fact-01kwjtjbnvb36dtaf2zrq806cj a mem:Fact ; mem:content "RDFS enforcement entailment (feature/rdfs-enforcement-entailment): always-on subclass+subproperty inference for SHACL and Policy. Epoch counters on Novelty bumped in apply_commit's routing loop (schema_epoch: subClassOf/subPropertyOf; shacl_epoch: sh:* predicates or rdf:type→SHACL-type/rdfs:Class/owl:Class) + two Arc caches on LedgerState carried across commits: SchemaHierarchyCache (core; keyed indexed-schema-t + schema_epoch) and a type-erased compiled-SHACL slot (+shacl_epoch +shapes_g_ids; data-only txns skip ShapeCompiler via ShaclEngine::from_shared_cache). RULE: enforcement uses COMMITTED hierarchy — same-txn schema does not entail (two-txn workflow, pinned by test)." ; mem:tag "enforcement" ; @@ -1477,6 +1493,80 @@ mem:fact-01kwjtjbnvb36dtaf2zrq806cj a mem:Fact ; mem:branch "feature/rdfs-enforcement-entailment" ; mem:createdAt "2026-07-03T01:47:52.379794+00:00"^^xsd:dateTime . +mem:fact-01kwktkb2r0ktxt5xaxvbwev3e a mem:Fact ; + mem:content "The \"query peer over HTTP\" model exists as server peer proxy mode: ProxyStorage + ProxyNameService wired via FlureeBuilder::memory().build_with(storage, NameServiceMode::ReadOnly); serving side is the StorageProxyBearer route group; GET /nameservice/snapshot filters to the token's fluree.storage.ledgers scope. ProxyStorage is mode-aware (feature/remote-mounts): ProxyReadMode::Raw fetches canonical CID-verified bytes via GET /storage/objects/{cid} (peer default; what makes indexed ledgers readable), ProxyReadMode::Filtered uses POST /storage/block FLKB negotiation. CRITICAL: no production code decodes FLKB — the filtered tier is transport-only until a leaf decoder exists on the read path. Remaining gaps: one upstream per server (no composite nameservice), no HTTP Range on ProxyStorage, no vended-credential/origin handoff." ; + mem:tag "auth" ; + mem:tag "federation" ; + mem:tag "nameservice" ; + mem:tag "peer-mode" ; + mem:tag "remote" ; + mem:tag "storage-proxy" ; + mem:scope mem:repo ; + mem:artifactRef "fluree-db-server/src/peer/proxy_nameservice.rs" ; + mem:artifactRef "fluree-db-server/src/peer/proxy_storage.rs" ; + mem:artifactRef "fluree-db-server/src/routes/nameservice_refs.rs" ; + mem:artifactRef "fluree-db-server/src/routes/storage_proxy.rs" ; + mem:artifactRef "fluree-db-server/src/state.rs" ; + mem:branch "feature/rdfs-enforcement-entailment" ; + mem:createdAt "2026-07-03T11:07:38.968674+00:00"^^xsd:dateTime ; + mem:rationale "Any future \"connect to external Fluree as a data source\" feature should extend peer proxy mode rather than invent a new graph-source kind; re-discovering this subsystem map cost four parallel explorations." . + +mem:constraint-01kwm1fjv37hjcfsga916mc5wv a mem:Constraint ; + mem:content "docs/design/ files must document permanent design — what exists and how it works — never phased plans, roadmaps, or implementation sequencing. Planning content goes in a temporary file elsewhere (scratchpad or uncommitted), not in docs/design/." ; + mem:tag "design-docs" ; + mem:tag "docs" ; + mem:tag "user-feedback" ; + mem:tag "workflow" ; + mem:scope mem:repo ; + mem:severity "must" ; + mem:branch "feature/remote-mounts" ; + mem:createdAt "2026-07-03T13:07:55.875908+00:00"^^xsd:dateTime ; + mem:rationale "User feedback when starting the remote-mounts feature: a proposed design doc had drifted toward a work plan (implementation slices/phases). Same spirit as the existing rule against plan labels in code comments and commit bodies." . + +mem:fact-01kwm585khra36js0r49n36k06 a mem:Fact ; + mem:content "Tests that run a peer-mode Fluree query against an in-process axum server must use #[tokio::test(flavor = \"multi_thread\")]: lazy dict-blob materialization during query execution bridges sync→async (block_on), which starves a current-thread runtime so the in-process server can never answer the HTTP fetch — the ProxyStorage request times out after its full 60s. See test_peer_proxy_fluree_queries_indexed_ledger." ; + mem:tag "gotcha" ; + mem:tag "peer-mode" ; + mem:tag "sync-bridge" ; + mem:tag "testing" ; + mem:tag "tokio" ; + mem:scope mem:repo ; + mem:artifactRef "fluree-db-server/tests/proxy_integration.rs" ; + mem:branch "feature/remote-mounts" ; + mem:createdAt "2026-07-03T14:13:47.249260+00:00"^^xsd:dateTime . + +mem:fact-01kwm9svy064yer4ek7rn251hh a mem:Fact ; + mem:content "Per-ledger serving posture = f:servingDefaults setting group (f:serveQuery/f:serveBlocks/f:publicVisibility, ledger-scoped, LedgerConfig.serving) enforced only on transaction-role servers: query gate in load_ledger_for_query (403 stable msg), blocks gate in storage_proxy/pack/commits (404). GOTCHA: @shared dict-blob addresses carry only the ledger NAME, so ProxyStorage derives a default-branch alias — any server-side per-ledger guard on block/object endpoints must branch-resolve DictBlob requests via list_branches (resolve_block_ledger in storage_proxy.rs) or non-main-branch peers break on dict fetches. Legacy per-branch dict layout ({name}/{branch}/index/objects/dicts) is also parsed by cid_and_ledger_from_address." ; + mem:tag "config-graph" ; + mem:tag "dict-blob" ; + mem:tag "gotcha" ; + mem:tag "peer-mode" ; + mem:tag "serving-defaults" ; + mem:tag "storage-proxy" ; + mem:scope mem:repo ; + mem:artifactRef "fluree-db-api/src/config_resolver.rs" ; + mem:artifactRef "fluree-db-core/src/ledger_config.rs" ; + mem:artifactRef "fluree-db-server/src/routes/serving.rs" ; + mem:artifactRef "fluree-db-server/src/routes/storage_proxy.rs" ; + mem:branch "feature/remote-mounts" ; + mem:createdAt "2026-07-03T15:33:21.472958+00:00"^^xsd:dateTime . + +mem:fact-01kwmb5yq11y50j3kbd7c5tne4 a mem:Fact ; + mem:content "Remote mounts: FlureeBuilder::with_remote_mount(RemoteMountSpec) applies in finalize_with_backend → CompositeNameService (prefix-routed reads, records localized to \"acme/inventory:main\", writes to mounts rejected, full publisher surface) + StorageBackend::Routed (namespace-prefix → mount backend at the content_store seam; admin ops = default only). ProxyStorage/ProxyNameService live in fluree-db-nameservice-sync (server peer/* re-exports); ProxyStorage::with_local_prefix strips the mount prefix from derived aliases. Transact writes to mounts fail at routed ProxyStorage (read-only) before the NS guard — the guard backstops create/branch/publish." ; + mem:tag "builder" ; + mem:tag "composite-nameservice" ; + mem:tag "peer-mode" ; + mem:tag "remote-mounts" ; + mem:tag "storage-backend" ; + mem:scope mem:repo ; + mem:artifactRef "fluree-db-api/src/lib.rs" ; + mem:artifactRef "fluree-db-core/src/storage.rs" ; + mem:artifactRef "fluree-db-nameservice-sync/src/proxy_storage.rs" ; + mem:artifactRef "fluree-db-nameservice/src/mount.rs" ; + mem:artifactRef "fluree-db-server/tests/proxy_integration.rs" ; + mem:branch "feature/remote-mounts" ; + mem:createdAt "2026-07-03T15:57:26.113125+00:00"^^xsd:dateTime . + mem:fact-01kvzn4mmmcvc7zz6bcswtw47c a mem:Fact ; mem:content "SPARQL property paths, branch feature/sparql-property-paths: closed ALL reported gaps PLUS inverse-in-composite (wired in BOTH sparql lower/path.rs AND query parse/lower.rs for JSON-LD parity). (1) ZeroOrOne p?: PathModifier::ZeroOrOne; BFS self+1hop. (2) Negated sets !: fresh ?__np{n} pred-var triple + FILTER(?p != IRI(...)); fwd/inv/mixed=UNION. (3) Nested modifiers (p*)*: collapse_modifiers (zero=any */?, unbounded=any +/*). (4) Composite-transitive (a/b)+ incl inverse steps (^a/b)+: PropertyPathPattern has predicates+first_inverse+sequence_steps:Vec; operator's read_step reads SPOT/POST by direction (opposite index when retreating); single_predicate() bails on composite. ONLY UNSUPPORTED (niche): nested transitive step in a composite unit (a+/b)+ — not exercised by any W3C test." ; mem:tag "negated-set" ; From 4eae7ba536ada0b506708c6c85d4c7554e75135f Mon Sep 17 00:00:00 2001 From: bplatz Date: Fri, 3 Jul 2026 13:46:59 -0400 Subject: [PATCH 5/9] feat(server,api,sync,cli): vended S3 credentials for peer block access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For S3-backed origins, hand authorized peers short-lived STS credentials scoped to a ledger's prefix so they read index content directly from S3 (native ranged reads, no origin bandwidth) instead of proxying every object through the origin's HTTP server. - fluree-db-api::vended_credentials: STS AssumeRole minting with a session policy narrowed to the ledger's name-level prefix (covers all branches + the @shared dict namespace, matching the all-or-nothing raw tier); s3:ListBucket is prefix-conditioned so missing keys stay 404 (the reader's legacy dict fallback depends on 404-vs-403); S3VendScope extraction from the connection config (single-bucket S3 only — split commit/index layouts are refused). - Server: GET /storage/credentials?ledger= behind the same guards as raw object serving (bearer scope, namespace guard, f:serveBlocks posture; 404 anti-leak). Config: --storage-vend-enabled, --storage-vend-role-arn, --storage-vend-ttl-secs (default 900, the STS minimum — grants outlive revocations until expiry, so short TTLs). - fluree-db-nameservice-sync (feature aws): grant fetch client, a ProvideCredentials impl that refreshes grants inside a 60s expiry margin (single-flight), and build_vended_s3_storage composing an S3Storage whose credentials auto-refresh; 404 means fall back to proxied reads. - CLI peer mode probes the endpoint and prefers direct S3 automatically, falling back to ProxyStorage. - Tests: LocalStack round trip (mint -> grant -> S3 reader -> CAS object read through the fluree address layer), wiremock refresh/404 paths, policy-shape unit tests, server 404 gate test. --- Cargo.lock | 4 + Cargo.toml | 1 + docs/api/endpoints.md | 54 +++ docs/cli/track.md | 2 +- docs/design/remote-mounts.md | 31 +- docs/operations/query-peers.md | 20 ++ fluree-db-api/Cargo.toml | 11 +- fluree-db-api/src/lib.rs | 2 + fluree-db-api/src/vended_credentials.rs | 289 +++++++++++++++ .../it_vended_credentials_testcontainers.rs | 188 ++++++++++ fluree-db-cli/Cargo.toml | 2 +- fluree-db-cli/src/context.rs | 67 +++- fluree-db-nameservice-sync/Cargo.toml | 10 + fluree-db-nameservice-sync/src/lib.rs | 2 + fluree-db-nameservice-sync/src/vended_s3.rs | 330 ++++++++++++++++++ fluree-db-server/src/config.rs | 24 ++ fluree-db-server/src/routes/mod.rs | 7 + fluree-db-server/src/routes/storage_proxy.rs | 73 ++++ fluree-db-server/src/state.rs | 71 ++++ fluree-db-server/tests/proxy_integration.rs | 31 ++ 20 files changed, 1201 insertions(+), 18 deletions(-) create mode 100644 fluree-db-api/src/vended_credentials.rs create mode 100644 fluree-db-api/tests/it_vended_credentials_testcontainers.rs create mode 100644 fluree-db-nameservice-sync/src/vended_s3.rs diff --git a/Cargo.lock b/Cargo.lock index 74fff4ecf3..06956c3848 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2433,6 +2433,7 @@ dependencies = [ "aws-config", "aws-sdk-dynamodb", "aws-sdk-s3", + "aws-sdk-sts", "base64 0.22.1", "bigdecimal 0.4.10", "bytes", @@ -2879,10 +2880,13 @@ version = "4.1.1" dependencies = [ "async-stream", "async-trait", + "aws-config", + "aws-credential-types", "bytes", "fluree-db-core", "fluree-db-nameservice", "fluree-db-novelty", + "fluree-db-storage-aws", "fluree-sse", "futures", "parking_lot", diff --git a/Cargo.toml b/Cargo.toml index bdbd4cc1b1..7e93cd4cb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,7 @@ tokio = { version = "1", features = ["full", "test-util"] } aws-config = "1.8.15" aws-sdk-s3 = "1.124.0" aws-sdk-dynamodb = "1.109.0" +aws-sdk-sts = "1.101.0" aws-credential-types = "1.2" aws-smithy-types = "1.4.7" once_cell = "1.19" diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index b2bf29f73e..b3f750c89d 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -900,6 +900,60 @@ curl -H "Authorization: Bearer $TOKEN" \ "http://localhost:8090/v1/fluree/storage/objects/bafybeig...leafCid?ledger=mydb:main" ``` +Single-range `Range: bytes=start-end` requests are honored with `206 Partial +Content` (`Content-Range` included); the full object is verified against the +CID before slicing. Unsupported Range forms fall back to a full `200` +response; a start at or past the object length returns `416`. + +### GET /storage/credentials + +Mint STS credentials scoped to a ledger's S3 prefix, so an authorized peer +reads index content **directly from S3** instead of proxying every object +through this server. See [Remote mounts and serving +tiers](../design/remote-mounts.md). + +**URL:** + +``` +GET /storage/credentials?ledger={ledger-id} +``` + +**Requirements (all must hold, else `404`):** + +- Server built with the `aws` feature and **single-bucket S3-backed storage** + (connection config) +- `--storage-vend-enabled` and `--storage-vend-role-arn ` configured +- Bearer token with `fluree.storage.*` scope covering the ledger +- The ledger's `f:serveBlocks` posture allows raw serving + +**Response Body** (200 OK): + +```json +{ + "access_key_id": "ASIA...", + "secret_access_key": "...", + "session_token": "...", + "expires_at_epoch_secs": 1751600000, + "bucket": "my-fluree-bucket", + "region": "us-east-1", + "key_prefix": "ledgers", + "scoped_prefix": "ledgers/mydb" +} +``` + +The grant is an STS `AssumeRole` session narrowed by a session policy to +`s3:GetObject` under the ledger's **name-level** prefix (covering all +branches and the shared dictionary namespace) plus prefix-conditioned +`s3:ListBucket`. TTL is `--storage-vend-ttl-secs` (default 900, the STS +minimum). + +**Revocation caveat:** a minted grant stays valid until it expires — token +revocations and `f:serveBlocks` changes take effect at grant expiry, so keep +TTLs short. + +Consumers (CLI peer mode, `fluree-db-nameservice-sync::vended_s3`) probe this +endpoint and fall back to proxied `/storage/objects` reads on `404`. + ## Nameservice Sync Endpoints Used by replication clients and peer instances to push ref updates, initialize diff --git a/docs/cli/track.md b/docs/cli/track.md index 110bc56c07..747f07289e 100644 --- a/docs/cli/track.md +++ b/docs/cli/track.md @@ -3,7 +3,7 @@ Track a remote ledger without storing local data. Tracked ledgers keep a lightweight record locally so you can use short aliases and the active-ledger shortcut. Two query modes are available: - **proxy** (default): reads and writes route to the remote server over HTTP — the remote executes your queries (its compute, row-level policy applied). -- **peer**: queries execute **locally** against index blocks fetched on demand from the remote's raw storage tier, CID-verified and cached on disk per remote (see `fluree cache`). Writes still forward to the remote over HTTP. Requires a bearer token with `fluree.storage.*` scope for the ledger — the remote serves its full contents, so this mode is only offered for ledgers you may read in full (see [Remote mounts and serving tiers](../design/remote-mounts.md)). +- **peer**: queries execute **locally** against index blocks fetched on demand from the remote's raw storage tier, CID-verified and cached on disk per remote (see `fluree cache`). Writes still forward to the remote over HTTP. Requires a bearer token with `fluree.storage.*` scope for the ledger — the remote serves its full contents, so this mode is only offered for ledgers you may read in full (see [Remote mounts and serving tiers](../design/remote-mounts.md)). When the remote vends S3 credentials (`GET /storage/credentials`), peer reads go directly to S3 automatically; otherwise blocks proxy through the remote's HTTP endpoints. Use `fluree remote ledgers ` to see which ledgers your token can access and which serving tiers (`query`, `blocks`) each offers. diff --git a/docs/design/remote-mounts.md b/docs/design/remote-mounts.md index 377a2fe559..4a6e11addf 100644 --- a/docs/design/remote-mounts.md +++ b/docs/design/remote-mounts.md @@ -118,6 +118,30 @@ selects by namespace prefix at exactly that point — so `LedgerState::load`, `BranchedContentStore` ancestry, and default-context reads all route with no changes. Admin operations (delete, list) apply only to the default backend. +### Direct S3 access: vended credentials + +When the origin's storage is S3, it can hand full-access consumers +short-lived credentials instead of proxying bytes: `GET +/storage/credentials?ledger=…` (behind the same token-scope and +`f:serveBlocks` guards as raw object serving) mints an STS `AssumeRole` +session narrowed by a session policy to the ledger's **name-level** S3 +prefix — one grant covers every branch plus the name-scoped `@shared` +dictionary namespace, matching the all-or-nothing raw tier. The response +carries the credentials plus everything a reader needs (`bucket`, `region`, +`endpoint`, `key_prefix`), and the consumer builds a normal S3 reader whose +credentials auto-refresh from the endpoint as grants approach expiry +(`fluree-db-nameservice-sync::vended_s3`). The CLI's peer mode probes the +endpoint and prefers direct S3 (native ranged reads, no origin bandwidth); +a 404 means "not vended here" and reads fall back to the HTTP proxy tier. + +Because everything fetched is CID-addressed, direct-from-S3 reads keep the +same integrity and cache-forever semantics as proxied ones. Two deliberate +limits: grants are only minted for **single-bucket** S3 layouts (a split +commit/index configuration would need a two-tier grant), and revocation is +expiry-bound — a minted grant outlives token revocation and `f:serveBlocks` +changes until its TTL lapses, so TTLs default to the STS minimum (15 +minutes). + ### Transport: `ProxyStorage` / `ProxyNameService` `fluree-db-nameservice-sync` (the HTTP client crate; the server re-exports @@ -216,6 +240,7 @@ added without reworking v1: reinterpretation of `fluree.storage.*` (which remains full-access). Similarly reserved: `f:publicVisibility` for an anonymous (public dataset) -tier, and origin/vended-credential handoff (`LedgerConfig.origins` already -models prioritized external origins — S3/IPFS/CDN — so a full-access -consumer could bypass the proxy entirely). +tier, and generalizing the vended-credential handoff into +`LedgerConfig.origins` (which already models prioritized external origins — +S3/IPFS/CDN — so consumers could discover origins from the record rather +than probing the credentials endpoint). diff --git a/docs/operations/query-peers.md b/docs/operations/query-peers.md index 4fa20a33f0..ac6d5e6d51 100644 --- a/docs/operations/query-peers.md +++ b/docs/operations/query-peers.md @@ -142,6 +142,26 @@ Partial Content response (`Content-Range` included). The server verifies the the same corruption guarantee; peers use this for leaflet-granular index reads instead of pulling whole objects. +#### `GET /v1/fluree/storage/credentials?ledger={ledger-id}` + +For S3-backed servers: mint short-lived STS credentials scoped to the +ledger's S3 prefix so the peer reads index content **directly from S3** +(native ranged reads, no origin bandwidth). Guarded exactly like raw object +serving. Enable with: + +- **`--storage-vend-enabled`** (`FLUREE_STORAGE_VEND_ENABLED`) +- **`--storage-vend-role-arn `** — IAM role the server assumes per + grant; the role's permissions are the ceiling, each grant is narrowed by a + session policy to the requested ledger's prefix. The server's own AWS + identity needs `sts:AssumeRole` on this role. +- **`--storage-vend-ttl-secs`** (default 900, the STS minimum). A minted + grant stays valid until expiry — token revocations and `f:serveBlocks` + changes take effect at grant expiry, so keep TTLs short. + +Requires single-bucket S3 storage (split commit/index bucket layouts are +refused). Peers probe this endpoint and fall back to proxied block reads on +404, so enabling/disabling it is transparent to consumers. + #### `POST /v1/fluree/storage/block` Fetch a block/blob by **CID** with policy mediation. The request includes the **ledger ID** so the server can authorize the request and derive the physical storage address internally. Currently supports: diff --git a/fluree-db-api/Cargo.toml b/fluree-db-api/Cargo.toml index 8eb964c09d..79399d8902 100644 --- a/fluree-db-api/Cargo.toml +++ b/fluree-db-api/Cargo.toml @@ -10,8 +10,9 @@ repository.workspace = true [features] default = ["native"] native = ["fluree-db-nameservice/native", "fluree-db-core/native", "fluree-db-query/native", "dep:sysinfo"] -# Enable AWS-backed storage support (S3, storage-backed nameservice). -aws = ["fluree-db-connection/aws", "dep:fluree-db-storage-aws"] +# Enable AWS-backed storage support (S3, storage-backed nameservice, +# vended-credential minting). +aws = ["fluree-db-connection/aws", "dep:fluree-db-storage-aws", "dep:aws-sdk-sts"] # Iceberg / R2RML graph source support (pulls in AWS SDK + native deps). The # Arrow columnar reader is the single graph-source read path (native predicate # pushdown: row-group skipping + exact row filtering). @@ -101,6 +102,7 @@ testcontainers = { version = "0.23", optional = true } aws-config = { workspace = true, optional = true } aws-sdk-s3 = { workspace = true, optional = true } aws-sdk-dynamodb = { workspace = true, optional = true } +aws-sdk-sts = { workspace = true, optional = true } # HTTP client for remote search service (feature-gated) reqwest = { workspace = true, optional = true } @@ -220,6 +222,11 @@ name = "it_storage_s3_testcontainers" path = "tests/it_storage_s3_testcontainers.rs" required-features = ["aws-testcontainers"] +[[test]] +name = "it_vended_credentials_testcontainers" +path = "tests/it_vended_credentials_testcontainers.rs" +required-features = ["aws", "aws-testcontainers"] + [[test]] name = "it_graph_source_vector" path = "tests/it_graph_source_vector.rs" diff --git a/fluree-db-api/src/lib.rs b/fluree-db-api/src/lib.rs index 2b3c48dcbc..3da5da41d2 100644 --- a/fluree-db-api/src/lib.rs +++ b/fluree-db-api/src/lib.rs @@ -87,6 +87,8 @@ pub mod tx_builder; pub mod validate; #[cfg(feature = "vector")] pub mod vector_worker; +#[cfg(feature = "aws")] +pub mod vended_credentials; pub mod view; pub mod wire; diff --git a/fluree-db-api/src/vended_credentials.rs b/fluree-db-api/src/vended_credentials.rs new file mode 100644 index 0000000000..68b4322ced --- /dev/null +++ b/fluree-db-api/src/vended_credentials.rs @@ -0,0 +1,289 @@ +//! Vended S3 credentials: mint STS credentials scoped to one ledger's +//! S3 prefix, so full-access peers read index content directly from S3 +//! instead of proxying bytes through the origin server. +//! +//! The scope is **name-level**: a grant for `inventory:main` covers +//! `{root_prefix}/inventory/*`, which includes every branch and the +//! name-scoped `@shared/dicts` namespace — matching the all-or-nothing +//! access model of the raw block tier (see +//! `docs/design/remote-mounts.md`). +//! +//! Revocation caveat: unlike per-request block serving, a vended grant is +//! valid until it expires. Serving-posture changes (`f:serveBlocks`) and +//! token revocations take effect at grant expiry — keep TTLs short. + +use crate::error::{ApiError, Result}; +use serde::{Deserialize, Serialize}; + +/// The S3 location a server serves a ledger from, used to scope grants. +#[derive(Debug, Clone)] +pub struct S3VendScope { + /// Bucket holding the ledger's CAS content. + pub bucket: String, + /// Root key prefix the storage backend prepends to every ledger path + /// (`S3StorageConfig.prefix`), if any. + pub root_prefix: Option, + /// Region the bucket lives in. `None` falls back to the ambient AWS + /// SDK region at mint time (consumers need a region to build a client). + pub region: Option, + /// Non-AWS endpoint override (LocalStack, MinIO), passed through to + /// consumers. + pub endpoint: Option, +} + +impl S3VendScope { + /// Extract a vend scope from a parsed connection config, when its index + /// storage is S3. + /// + /// Returns `None` for non-S3 storage, and — for now — for split + /// commit/index buckets (a grant would need to cover both tiers; refuse + /// rather than mint a grant that can't read commits). + pub fn from_connection_config(config: &fluree_db_connection::ConnectionConfig) -> Option { + use fluree_db_connection::config::StorageType; + + let StorageType::S3(index) = &config.index_storage.storage_type else { + return None; + }; + + if let Some(commit) = &config.commit_storage { + if let StorageType::S3(commit_s3) = &commit.storage_type { + if commit_s3.bucket != index.bucket || commit_s3.prefix != index.prefix { + tracing::warn!( + "vended credentials: split commit/index S3 storage is not supported; \ + credential vending disabled" + ); + return None; + } + } + } + + Some(Self { + bucket: index.bucket.to_string(), + root_prefix: index.prefix.as_ref().map(std::string::ToString::to_string), + region: None, + endpoint: index + .endpoint + .as_ref() + .map(std::string::ToString::to_string), + }) + } +} + +/// A minted, ledger-scoped S3 grant — the wire response of +/// `GET /storage/credentials`. +/// +/// Consumers deserialize this shape in `fluree-db-nameservice-sync`; keep +/// additions backward-compatible (new fields must be optional). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VendedS3Grant { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + /// Grant expiry as Unix epoch seconds. + pub expires_at_epoch_secs: i64, + pub bucket: String, + pub region: String, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub endpoint: Option, + /// Root key prefix to configure on the consumer's S3 storage + /// (`S3StorageConfig.prefix`) so address→key mapping matches the origin. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub key_prefix: Option, + /// The key prefix this grant is actually scoped to + /// (`{key_prefix}/{ledger_name}`) — informational. + pub scoped_prefix: String, +} + +/// The key prefix a grant for `ledger_id` is scoped to: the ledger's +/// name-level prefix under the backend's root prefix. +pub fn ledger_scope_prefix(root_prefix: Option<&str>, ledger_id: &str) -> String { + let name = ledger_id.split(':').next().unwrap_or(ledger_id); + match root_prefix { + Some(root) if !root.is_empty() => format!("{}/{name}", root.trim_end_matches('/')), + _ => name.to_string(), + } +} + +/// Build the STS session policy restricting the grant to read access under +/// the ledger's prefix. +/// +/// `s3:ListBucket` (prefix-conditioned) is included so missing keys surface +/// as 404 rather than 403 — the reader's not-found semantics (e.g. the +/// legacy dict-address fallback) depend on the distinction. +pub fn session_policy_json(bucket: &str, scoped_prefix: &str) -> String { + serde_json::json!({ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "GetLedgerObjects", + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": format!("arn:aws:s3:::{bucket}/{scoped_prefix}/*"), + }, + { + "Sid": "ListLedgerPrefix", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": format!("arn:aws:s3:::{bucket}"), + "Condition": { "StringLike": { "s3:prefix": format!("{scoped_prefix}/*") } }, + } + ] + }) + .to_string() +} + +/// Sanitize an identity into a valid STS role-session name +/// (`[\w+=,.@-]`, 2–64 chars). +fn session_name(identity: Option<&str>) -> String { + let raw = identity.unwrap_or("fluree-peer"); + let mut name: String = raw + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '+' | '=' | ',' | '.' | '@' | '-' | '_') { + c + } else { + '-' + } + }) + .take(64) + .collect(); + if name.len() < 2 { + name = "fluree-peer".to_string(); + } + name +} + +/// Mint a ledger-scoped S3 grant via STS `AssumeRole`. +/// +/// The assumed role's own permissions are the ceiling; the session policy +/// narrows them to the ledger's prefix. `ttl_secs` is clamped to the STS +/// minimum (900s); the role's `MaxSessionDuration` is the upper bound, +/// surfacing as an STS error if exceeded. +pub async fn mint_scoped_credentials( + sts: &aws_sdk_sts::Client, + role_arn: &str, + scope: &S3VendScope, + region: &str, + ledger_id: &str, + ttl_secs: i32, + identity: Option<&str>, +) -> Result { + let scoped_prefix = ledger_scope_prefix(scope.root_prefix.as_deref(), ledger_id); + let policy = session_policy_json(&scope.bucket, &scoped_prefix); + + let assumed = sts + .assume_role() + .role_arn(role_arn) + .role_session_name(session_name(identity)) + .policy(policy) + .duration_seconds(ttl_secs.max(900)) + .send() + .await + .map_err(|e| ApiError::internal(format!("STS AssumeRole failed: {e}")))?; + + let creds = assumed + .credentials() + .ok_or_else(|| ApiError::internal("STS AssumeRole returned no credentials"))?; + + Ok(VendedS3Grant { + access_key_id: creds.access_key_id().to_string(), + secret_access_key: creds.secret_access_key().to_string(), + session_token: creds.session_token().to_string(), + expires_at_epoch_secs: creds.expiration().secs(), + bucket: scope.bucket.clone(), + region: region.to_string(), + endpoint: scope.endpoint.clone(), + key_prefix: scope.root_prefix.clone(), + scoped_prefix, + }) +} + +/// Mint a grant using the process-global ambient AWS SDK configuration +/// (credentials chain + region), resolving the scope's region against it. +pub async fn mint_scoped_credentials_ambient( + role_arn: &str, + scope: &S3VendScope, + ledger_id: &str, + ttl_secs: i32, + identity: Option<&str>, +) -> Result { + let sdk = fluree_db_connection::aws::get_or_init_sdk_config() + .await + .map_err(|e| ApiError::internal(format!("AWS SDK config: {e}")))?; + let region = scope + .region + .clone() + .or_else(|| sdk.region().map(std::string::ToString::to_string)) + .ok_or_else(|| { + ApiError::internal("no AWS region configured for credential vending (set AWS_REGION)") + })?; + let sts = aws_sdk_sts::Client::new(sdk); + mint_scoped_credentials( + &sts, role_arn, scope, ®ion, ledger_id, ttl_secs, identity, + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scope_prefix_covers_name_level() { + assert_eq!( + ledger_scope_prefix(Some("ledgers"), "inventory:main"), + "ledgers/inventory" + ); + assert_eq!(ledger_scope_prefix(None, "inventory:main"), "inventory"); + assert_eq!(ledger_scope_prefix(Some(""), "inventory"), "inventory"); + assert_eq!( + ledger_scope_prefix(Some("root/"), "inv:feature"), + "root/inv" + ); + } + + #[test] + fn session_policy_scopes_get_and_list() { + let policy = session_policy_json("my-bucket", "ledgers/inventory"); + let parsed: serde_json::Value = serde_json::from_str(&policy).unwrap(); + let statements = parsed["Statement"].as_array().unwrap(); + assert_eq!(statements.len(), 2); + assert_eq!( + statements[0]["Resource"], + "arn:aws:s3:::my-bucket/ledgers/inventory/*" + ); + assert_eq!(statements[1]["Resource"], "arn:aws:s3:::my-bucket"); + assert_eq!( + statements[1]["Condition"]["StringLike"]["s3:prefix"], + "ledgers/inventory/*" + ); + } + + #[test] + fn session_name_sanitizes() { + assert_eq!(session_name(Some("did:key:z6Mk/abc")), "did-key-z6Mk-abc"); + assert_eq!(session_name(Some("x")), "fluree-peer"); + assert_eq!(session_name(None), "fluree-peer"); + assert!(session_name(Some(&"a".repeat(100))).len() <= 64); + } + + #[test] + fn grant_round_trips_json() { + let grant = VendedS3Grant { + access_key_id: "AKIA".into(), + secret_access_key: "secret".into(), + session_token: "token".into(), + expires_at_epoch_secs: 1_700_000_000, + bucket: "b".into(), + region: "us-east-1".into(), + endpoint: None, + key_prefix: Some("ledgers".into()), + scoped_prefix: "ledgers/inventory".into(), + }; + let json = serde_json::to_string(&grant).unwrap(); + let back: VendedS3Grant = serde_json::from_str(&json).unwrap(); + assert_eq!(back.bucket, "b"); + assert_eq!(back.key_prefix.as_deref(), Some("ledgers")); + assert!(!json.contains("endpoint"), "None endpoint omitted: {json}"); + } +} diff --git a/fluree-db-api/tests/it_vended_credentials_testcontainers.rs b/fluree-db-api/tests/it_vended_credentials_testcontainers.rs new file mode 100644 index 0000000000..9c6ae76f5e --- /dev/null +++ b/fluree-db-api/tests/it_vended_credentials_testcontainers.rs @@ -0,0 +1,188 @@ +//! Vended-credential integration test using testcontainers + LocalStack. +//! +//! Round-trips the full grant flow: mint STS credentials scoped to a +//! ledger's prefix, build an S3 reader from the grant, and read a CAS +//! object with it. +//! +//! Note: LocalStack community edition does not enforce IAM session +//! policies, so this validates minting and grant usability — the +//! prefix-scoping *policy shape* is covered by unit tests in +//! `fluree_db_api::vended_credentials`. +//! +//! Run (requires Docker): +//! cargo test -p fluree-db-api --features aws,aws-testcontainers \ +//! --test it_vended_credentials_testcontainers -- --nocapture + +#![cfg(all(feature = "aws", feature = "aws-testcontainers"))] + +use aws_config::meta::region::RegionProviderChain; +use fluree_db_api::vended_credentials::{mint_scoped_credentials, S3VendScope}; +use fluree_db_core::StorageRead; +use fluree_db_storage_aws::{S3Config, S3Storage}; +use fs2::FileExt; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::{runners::AsyncRunner, GenericImage, ImageExt}; + +const LOCALSTACK_EDGE_PORT: u16 = 4566; +const REGION: &str = "us-east-1"; + +struct LocalstackTestLock { + _file: std::fs::File, +} + +fn set_test_aws_env() { + std::env::set_var("AWS_ACCESS_KEY_ID", "test"); + std::env::set_var("AWS_SECRET_ACCESS_KEY", "test"); + std::env::set_var("AWS_REGION", REGION); + std::env::set_var("AWS_DEFAULT_REGION", REGION); + std::env::set_var("AWS_EC2_METADATA_DISABLED", "true"); +} + +async fn sdk_config_for_localstack(endpoint: &str) -> aws_config::SdkConfig { + set_test_aws_env(); + let region_provider = RegionProviderChain::default_provider().or_else(REGION); + aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(region_provider) + .endpoint_url(endpoint) + .load() + .await +} + +async fn acquire_localstack_test_lock() -> LocalstackTestLock { + tokio::task::spawn_blocking(|| { + let lock_path = std::env::temp_dir().join("fluree-localstack-tests.lock"); + let lock_file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .unwrap_or_else(|e| panic!("open LocalStack test lock {}: {e}", lock_path.display())); + lock_file + .lock_exclusive() + .expect("acquire LocalStack test lock"); + LocalstackTestLock { _file: lock_file } + }) + .await + .expect("lock task") +} + +async fn start_localstack( + services: &str, +) -> ( + LocalstackTestLock, + testcontainers::ContainerAsync, + String, +) { + let lock = acquire_localstack_test_lock().await; + let image = GenericImage::new("localstack/localstack", "4.4") + .with_exposed_port(LOCALSTACK_EDGE_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stdout("Ready.")) + .with_env_var("SERVICES", services) + .with_env_var("DEFAULT_REGION", REGION) + .with_env_var("SKIP_SSL_CERT_DOWNLOAD", "1") + .with_startup_timeout(Duration::from_secs(300)); + let container = image + .start() + .await + .expect("LocalStack started (Docker must be running)"); + let host_port = container + .get_host_port_ipv4(LOCALSTACK_EDGE_PORT) + .await + .expect("LocalStack edge port"); + let endpoint = format!("http://127.0.0.1:{host_port}"); + (lock, container, endpoint) +} + +#[tokio::test] +async fn vended_grant_mints_and_reads_ledger_objects() { + let (_lock, _container, endpoint) = start_localstack("s3,sts,iam").await; + let sdk_config = sdk_config_for_localstack(&endpoint).await; + + // Seed a bucket with one CAS object under the ledger's prefix. + let bucket = "vend-test-bucket"; + let s3_admin = aws_sdk_s3::Client::new(&sdk_config); + s3_admin + .create_bucket() + .bucket(bucket) + .send() + .await + .expect("create bucket"); + let object_bytes = b"canonical commit bytes".to_vec(); + s3_admin + .put_object() + .bucket(bucket) + .key("ledgers/inv/main/commit/test.fc") + .body(object_bytes.clone().into()) + .send() + .await + .expect("seed object"); + + // Mint a grant scoped to the ledger's name-level prefix. + let sts = aws_sdk_sts::Client::new(&sdk_config); + let scope = S3VendScope { + bucket: bucket.to_string(), + root_prefix: Some("ledgers".to_string()), + region: Some(REGION.to_string()), + endpoint: Some(endpoint.clone()), + }; + let grant = mint_scoped_credentials( + &sts, + "arn:aws:iam::000000000000:role/fluree-vend-test", + &scope, + REGION, + "inv:main", + 900, + Some("did:key:z6MkTest"), + ) + .await + .expect("mint grant"); + + assert_eq!(grant.scoped_prefix, "ledgers/inv"); + assert_eq!(grant.bucket, bucket); + assert!(!grant.session_token.is_empty(), "STS session token present"); + let now_secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() as i64; + assert!( + grant.expires_at_epoch_secs > now_secs + 600, + "grant expiry should be well in the future: {} vs {now_secs}", + grant.expires_at_epoch_secs + ); + + // Build a consumer S3 reader from the grant (the same construction + // `fluree-db-nameservice-sync::vended_s3` performs) and read the object + // through the fluree address layer. + let creds = aws_sdk_s3::config::Credentials::new( + grant.access_key_id.clone(), + grant.secret_access_key.clone(), + Some(grant.session_token.clone()), + None, + "vended-test", + ); + let consumer_sdk = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .credentials_provider(creds) + .region(aws_config::Region::new(REGION)) + .endpoint_url(&endpoint) + .load() + .await; + let consumer = S3Storage::new( + &consumer_sdk, + S3Config { + bucket: grant.bucket.clone(), + prefix: grant.key_prefix.clone(), + endpoint: grant.endpoint.clone(), + ..Default::default() + }, + ) + .await + .expect("consumer storage"); + + let bytes = consumer + .read_bytes("fluree:s3://inv/main/commit/test.fc") + .await + .expect("vended read"); + assert_eq!(bytes, object_bytes); +} diff --git a/fluree-db-cli/Cargo.toml b/fluree-db-cli/Cargo.toml index c7cca63563..a1da94f7af 100644 --- a/fluree-db-cli/Cargo.toml +++ b/fluree-db-cli/Cargo.toml @@ -23,7 +23,7 @@ path = "src/lib.rs" default = ["server", "iceberg", "shacl", "aws"] server = ["dep:fluree-db-server"] iceberg = ["fluree-db-api/iceberg"] -aws = ["fluree-db-server/aws"] +aws = ["fluree-db-server/aws", "fluree-db-nameservice-sync/aws"] # SHACL constraint validation at transaction time shacl = ["fluree-db-api/shacl"] # Use mimalloc as the global allocator (better multicore allocation throughput diff --git a/fluree-db-cli/src/context.rs b/fluree-db-cli/src/context.rs index 3f3a160ab6..e2720419ca 100644 --- a/fluree-db-cli/src/context.rs +++ b/fluree-db-cli/src/context.rs @@ -342,14 +342,19 @@ async fn build_tracked_mode( } /// Build a [`QueryTarget::Peer`]: an embedded Fluree whose reads go to the -/// remote's raw storage tier (`ProxyStorage` in Raw mode, CID-verified) with -/// a persistent per-remote disk cache for index artifacts. +/// remote's raw storage tier with a persistent per-remote disk cache for +/// index artifacts. +/// +/// Storage negotiation: when the remote vends S3 credentials for the ledger +/// (`GET /storage/credentials`), reads go directly to S3 with auto-refreshed +/// scoped credentials; otherwise blocks proxy through the remote's HTTP +/// endpoints (`ProxyStorage` in Raw mode, CID-verified). async fn build_peer_target( store: &TomlSyncConfigStore, tracked: &TrackedLedgerConfig, local_alias: &str, ) -> CliResult { - use fluree_db_api::{LedgerManagerConfig, NameServiceMode}; + use fluree_db_api::{Fluree, LedgerManagerConfig, NameServiceMode}; use fluree_db_nameservice_sync::{ProxyNameService, ProxyReadMode, ProxyStorage}; let (base_url, auth) = tracked_remote_http(store, tracked, local_alias).await?; @@ -363,11 +368,6 @@ async fn build_peer_target( })?; let client = build_client_from_auth(&base_url, &auth); - // Remote base URLs are API bases (ending in `/fluree`), so use the - // api-base constructors rather than the server-root ones. - let storage = ProxyStorage::from_api_base(base_url.clone(), token.clone(), ProxyReadMode::Raw); - let ns = ProxyNameService::from_api_base(base_url, token); - // Persistent, per-remote artifact cache. Everything cached is // content-addressed and immutable, so entries never invalidate; the // nameservice head lookup (verify_freshness_on_cache_hit) is the only @@ -378,9 +378,54 @@ async fn build_peer_target( ..Default::default() }; - let fluree = FlureeBuilder::memory() - .with_ledger_cache_config(cache_config) - .build_with(storage, NameServiceMode::ReadOnly(std::sync::Arc::new(ns))); + fn assemble( + cache_config: LedgerManagerConfig, + storage: impl fluree_db_api::Storage + 'static, + ns: ProxyNameService, + ) -> Fluree { + FlureeBuilder::memory() + .with_ledger_cache_config(cache_config) + .build_with(storage, NameServiceMode::ReadOnly(std::sync::Arc::new(ns))) + } + + // Remote base URLs are API bases (ending in `/fluree`), so use the + // api-base constructors rather than the server-root ones. + let ns = ProxyNameService::from_api_base(base_url.clone(), token.clone()); + + #[cfg(feature = "aws")] + let vended = match fluree_db_nameservice_sync::vended_s3::build_vended_s3_storage( + base_url.clone(), + token.clone(), + tracked.remote_alias.clone(), + ) + .await + { + Ok(storage) => storage, + Err(e) => { + tracing::debug!(error = %e, "vended credential probe failed; using proxied reads"); + None + } + }; + #[cfg(not(feature = "aws"))] + let vended: Option = None; + + let fluree = match vended { + #[cfg(feature = "aws")] + Some(s3) => { + eprintln!( + " {} reading '{}' directly from S3 (vended credentials)", + "notice:".dimmed(), + tracked.remote_alias + ); + assemble(cache_config, s3, ns) + } + #[cfg(not(feature = "aws"))] + Some(_) => unreachable!(), + None => { + let storage = ProxyStorage::from_api_base(base_url, token, ProxyReadMode::Raw); + assemble(cache_config, storage, ns) + } + }; Ok(QueryTarget::Peer { fluree: Box::new(fluree), diff --git a/fluree-db-nameservice-sync/Cargo.toml b/fluree-db-nameservice-sync/Cargo.toml index 45b51e8a27..9952446118 100644 --- a/fluree-db-nameservice-sync/Cargo.toml +++ b/fluree-db-nameservice-sync/Cargo.toml @@ -26,6 +26,16 @@ parking_lot = "0.12" urlencoding = { workspace = true } async-stream = "0.3" +# Vended S3 credentials consumer (feature "aws") +fluree-db-storage-aws = { path = "../fluree-db-storage-aws", optional = true, default-features = false, features = ["s3"] } +aws-config = { workspace = true, optional = true } +aws-credential-types = { workspace = true, optional = true } + +[features] +# Consume vended S3 credentials: build an S3 reader from a grant minted by +# a remote server's GET /storage/credentials endpoint. +aws = ["dep:fluree-db-storage-aws", "dep:aws-config", "dep:aws-credential-types"] + [dev-dependencies] fluree-db-core = { path = "../fluree-db-core" } tokio = { workspace = true, features = ["rt", "macros", "sync", "test-util"] } diff --git a/fluree-db-nameservice-sync/src/lib.rs b/fluree-db-nameservice-sync/src/lib.rs index 725252ab6e..0df9698657 100644 --- a/fluree-db-nameservice-sync/src/lib.rs +++ b/fluree-db-nameservice-sync/src/lib.rs @@ -29,6 +29,8 @@ pub mod pack_client; pub mod proxy_nameservice; pub mod proxy_storage; mod server_sse; +#[cfg(feature = "aws")] +pub mod vended_s3; pub mod watch; pub mod watch_poll; pub mod watch_sse; diff --git a/fluree-db-nameservice-sync/src/vended_s3.rs b/fluree-db-nameservice-sync/src/vended_s3.rs new file mode 100644 index 0000000000..95ad9fd5d9 --- /dev/null +++ b/fluree-db-nameservice-sync/src/vended_s3.rs @@ -0,0 +1,330 @@ +//! Consume vended S3 credentials: fetch a ledger-scoped grant from a remote +//! server's `GET /storage/credentials` endpoint and build an S3 reader from +//! it, refreshing credentials automatically as grants expire. +//! +//! This is the direct-from-S3 fast path for peer mode: the origin hands out +//! short-lived STS credentials scoped to the ledger's prefix, and the +//! consumer reads canonical CAS bytes (native ranged reads included) without +//! proxying every object through the origin's HTTP server. A 404 from the +//! endpoint means vending is unavailable — callers fall back to +//! [`ProxyStorage`](crate::proxy_storage::ProxyStorage). + +use aws_credential_types::provider::error::CredentialsError; +use aws_credential_types::provider::{future, ProvideCredentials}; +use aws_credential_types::Credentials; +use fluree_db_storage_aws::{S3Config, S3Storage}; +use reqwest::{Client, StatusCode}; +use serde::Deserialize; +use std::fmt::Debug; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::Mutex; + +use crate::error::{Result, SyncError}; + +/// Refresh grants this long before their stated expiry. +const REFRESH_MARGIN: Duration = Duration::from_secs(60); + +/// A ledger-scoped S3 grant as served by `GET /storage/credentials`. +/// +/// Mirrors the server's `VendedS3Grant` wire shape (fluree-db-api); optional +/// fields default so older servers stay parseable. +#[derive(Debug, Clone, Deserialize)] +pub struct VendedS3Grant { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + /// Grant expiry as Unix epoch seconds. + pub expires_at_epoch_secs: i64, + pub bucket: String, + pub region: String, + #[serde(default)] + pub endpoint: Option, + /// Root key prefix to configure on the S3 reader so address→key mapping + /// matches the origin. + #[serde(default)] + pub key_prefix: Option, + /// The prefix the grant is scoped to (informational). + #[serde(default)] + pub scoped_prefix: Option, +} + +impl VendedS3Grant { + fn expires_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.expires_at_epoch_secs.max(0) as u64) + } + + fn needs_refresh(&self) -> bool { + SystemTime::now() + REFRESH_MARGIN >= self.expires_at() + } + + fn to_credentials(&self) -> Credentials { + Credentials::new( + self.access_key_id.clone(), + self.secret_access_key.clone(), + Some(self.session_token.clone()), + Some(self.expires_at()), + "fluree-vended-credentials", + ) + } +} + +/// HTTP client for the vended-credentials endpoint. +#[derive(Clone)] +pub struct VendedCredentialsClient { + client: Client, + api_base: String, + token: String, + ledger: String, +} + +impl Debug for VendedCredentialsClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VendedCredentialsClient") + .field("api_base", &self.api_base) + .field("ledger", &self.ledger) + .finish_non_exhaustive() + } +} + +impl VendedCredentialsClient { + /// Create a client for one ledger's grants. `api_base` is the full API + /// base URL (e.g. `https://data.example.com/v1/fluree`). + pub fn new( + api_base: impl Into, + token: impl Into, + ledger: impl Into, + ) -> Self { + Self { + client: Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .expect("Failed to create vended credentials client"), + api_base: api_base.into().trim_end_matches('/').to_string(), + token: token.into(), + ledger: ledger.into(), + } + } + + /// Fetch a grant. `Ok(None)` means the server doesn't vend credentials + /// (endpoint absent, disabled, out of scope, or non-S3 storage) — + /// callers fall back to proxied block reads. + pub async fn fetch(&self) -> Result> { + let response = self + .client + .get(format!("{}/storage/credentials", self.api_base)) + .query(&[("ledger", self.ledger.as_str())]) + .bearer_auth(&self.token) + .send() + .await + .map_err(|e| SyncError::Remote(format!("vended credentials request failed: {e}")))?; + + match response.status() { + StatusCode::OK => { + let grant: VendedS3Grant = response.json().await.map_err(|e| { + SyncError::Remote(format!("vended credentials response parse failed: {e}")) + })?; + Ok(Some(grant)) + } + StatusCode::NOT_FOUND => Ok(None), + status => Err(SyncError::Remote(format!( + "vended credentials request rejected: {status}" + ))), + } + } +} + +/// AWS credentials provider that refreshes grants from the vending endpoint +/// as they approach expiry. +#[derive(Debug)] +pub struct RefreshingVendedProvider { + client: VendedCredentialsClient, + cached: Arc>, +} + +impl RefreshingVendedProvider { + pub fn new(client: VendedCredentialsClient, initial: VendedS3Grant) -> Self { + Self { + client, + cached: Arc::new(Mutex::new(initial)), + } + } + + async fn current_credentials(&self) -> std::result::Result { + // Holding the lock across the refresh single-flights concurrent + // refreshes; waiters then read the fresh grant. + let mut cached = self.cached.lock().await; + if cached.needs_refresh() { + match self.client.fetch().await { + Ok(Some(grant)) => *cached = grant, + Ok(None) => { + return Err(CredentialsError::provider_error( + "vending endpoint no longer offers credentials for this ledger", + )); + } + Err(e) => return Err(CredentialsError::provider_error(e)), + } + } + Ok(cached.to_credentials()) + } +} + +impl ProvideCredentials for RefreshingVendedProvider { + fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> + where + Self: 'a, + { + future::ProvideCredentials::new(self.current_credentials()) + } +} + +/// Build an S3 reader for one remote ledger from vended credentials. +/// +/// Fetches an initial grant (returning `Ok(None)` when the server doesn't +/// vend — fall back to proxied reads), then constructs an +/// [`S3Storage`] whose credentials refresh automatically via +/// [`RefreshingVendedProvider`]. +pub async fn build_vended_s3_storage( + api_base: impl Into, + token: impl Into, + ledger: impl Into, +) -> Result> { + let client = VendedCredentialsClient::new(api_base, token, ledger); + let Some(grant) = client.fetch().await? else { + return Ok(None); + }; + + let s3_config = S3Config { + bucket: grant.bucket.clone(), + prefix: grant.key_prefix.clone(), + endpoint: grant.endpoint.clone(), + ..Default::default() + }; + let region = aws_config::Region::new(grant.region.clone()); + let provider = RefreshingVendedProvider::new(client, grant); + + let sdk_config = aws_config::SdkConfig::builder() + .credentials_provider( + aws_credential_types::provider::SharedCredentialsProvider::new(provider), + ) + .region(region) + .behavior_version(aws_config::BehaviorVersion::latest()) + .build(); + + let storage = S3Storage::new(&sdk_config, s3_config) + .await + .map_err(|e| SyncError::Remote(format!("vended S3 storage init failed: {e}")))?; + Ok(Some(storage)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn grant(expires_in_secs: i64) -> VendedS3Grant { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() as i64; + VendedS3Grant { + access_key_id: "AKIA".into(), + secret_access_key: "secret".into(), + session_token: "token".into(), + expires_at_epoch_secs: now + expires_in_secs, + bucket: "b".into(), + region: "us-east-1".into(), + endpoint: None, + key_prefix: Some("ledgers".into()), + scoped_prefix: Some("ledgers/inv".into()), + } + } + + #[test] + fn refresh_margin_applies() { + assert!(grant(30).needs_refresh(), "inside the 60s margin"); + assert!(!grant(600).needs_refresh(), "well before expiry"); + assert!(grant(-10).needs_refresh(), "already expired"); + } + + #[test] + fn grant_parses_minimal_wire_shape() { + // Older/newer servers may omit optional fields. + let json = r#"{ + "access_key_id": "AKIA", + "secret_access_key": "s", + "session_token": "t", + "expires_at_epoch_secs": 1700000000, + "bucket": "b", + "region": "us-east-1", + "scoped_prefix": "inv" + }"#; + let grant: VendedS3Grant = serde_json::from_str(json).unwrap(); + assert!(grant.key_prefix.is_none()); + assert!(grant.endpoint.is_none()); + } + + fn grant_json(access_key: &str, expires_in_secs: i64) -> serde_json::Value { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() as i64; + serde_json::json!({ + "access_key_id": access_key, + "secret_access_key": "s", + "session_token": "t", + "expires_at_epoch_secs": now + expires_in_secs, + "bucket": "b", + "region": "us-east-1", + "scoped_prefix": "ledgers/inv" + }) + } + + #[tokio::test] + async fn fetch_returns_none_on_404() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/storage/credentials")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + + let client = VendedCredentialsClient::new(server.uri(), "tok", "inv:main"); + let result = client.fetch().await.expect("fetch should not error"); + assert!(result.is_none(), "404 means vending unavailable"); + } + + #[tokio::test] + async fn provider_refreshes_expiring_grant() { + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/storage/credentials")) + .and(query_param("ledger", "inv:main")) + .respond_with(ResponseTemplate::new(200).set_body_json(grant_json("AKIA-FRESH", 900))) + .expect(1) + .mount(&server) + .await; + + let client = VendedCredentialsClient::new(server.uri(), "tok", "inv:main"); + // Seed with a grant inside the refresh margin: the first + // provide_credentials must refetch. + let expiring: VendedS3Grant = serde_json::from_value(grant_json("AKIA-STALE", 10)).unwrap(); + let provider = RefreshingVendedProvider::new(client, expiring); + + let creds = provider + .provide_credentials() + .await + .expect("refresh should succeed"); + assert_eq!(creds.access_key_id(), "AKIA-FRESH"); + + // A fresh grant is served from cache — the mock's expect(1) verifies + // no second request happens. + let creds = provider.provide_credentials().await.expect("cached grant"); + assert_eq!(creds.access_key_id(), "AKIA-FRESH"); + } +} diff --git a/fluree-db-server/src/config.rs b/fluree-db-server/src/config.rs index 9d79307073..eb7c6df0ca 100644 --- a/fluree-db-server/src/config.rs +++ b/fluree-db-server/src/config.rs @@ -654,6 +654,26 @@ pub struct ServerConfig { #[arg(long, env = "FLUREE_STORAGE_PROXY_INSECURE", hide = true)] pub storage_proxy_insecure_accept_any_issuer: bool, + // === Vended S3 credentials (transaction server, S3-backed storage) === + /// Enable GET /storage/credentials: mint STS credentials scoped to a + /// ledger's S3 prefix so authorized peers read index content directly + /// from S3 instead of proxying bytes through this server. Requires + /// S3-backed storage (connection config) and --storage-vend-role-arn. + #[arg(long, env = "FLUREE_STORAGE_VEND_ENABLED")] + pub storage_vend_enabled: bool, + + /// IAM role ARN to assume when minting vended credentials. The role's + /// permissions are the ceiling; each grant is narrowed to the requested + /// ledger's prefix by a session policy. + #[arg(long, env = "FLUREE_STORAGE_VEND_ROLE_ARN")] + pub storage_vend_role_arn: Option, + + /// Vended credential TTL in seconds (STS minimum 900). Serving-posture + /// changes and token revocations take effect at grant expiry, so keep + /// this short. + #[arg(long, env = "FLUREE_STORAGE_VEND_TTL_SECS", default_value_t = 900)] + pub storage_vend_ttl_secs: i32, + // === Peer storage access mode options === /// Storage access mode for peer: shared (direct) or proxy (through tx server) #[arg( @@ -826,6 +846,10 @@ impl Default for ServerConfig { storage_proxy_default_policy_class: None, storage_proxy_debug_headers: false, storage_proxy_insecure_accept_any_issuer: false, + // Vended credential defaults + storage_vend_enabled: false, + storage_vend_role_arn: None, + storage_vend_ttl_secs: 900, // Peer storage access mode defaults storage_access_mode: StorageAccessMode::Shared, storage_proxy_token: None, diff --git a/fluree-db-server/src/routes/mod.rs b/fluree-db-server/src/routes/mod.rs index 6fe0a10b26..9739eda6c6 100644 --- a/fluree-db-server/src/routes/mod.rs +++ b/fluree-db-server/src/routes/mod.rs @@ -264,6 +264,13 @@ pub fn build_router(state: Arc) -> Router { .route("/subscribe", get(stubs::subscribe)) .route("/remote/:path", get(stubs::remote).post(stubs::remote)); + // Vended S3 credentials (absent without the aws feature → 404) + #[cfg(feature = "aws")] + let v1 = v1.route( + "/storage/credentials", + get(storage_proxy::get_vended_credentials), + ); + // SHACL validation report (read endpoint; see routes/validate.rs) #[cfg(feature = "shacl")] let v1 = v1.route( diff --git a/fluree-db-server/src/routes/storage_proxy.rs b/fluree-db-server/src/routes/storage_proxy.rs index 2371cff59c..98ec6252ba 100644 --- a/fluree-db-server/src/routes/storage_proxy.rs +++ b/fluree-db-server/src/routes/storage_proxy.rs @@ -230,6 +230,79 @@ pub struct BlockRequest { pub ledger: String, } +// ============================================================================ +// Vended Credentials +// ============================================================================ + +/// Query params for the vended-credentials endpoint. +#[cfg(feature = "aws")] +#[derive(Debug, Deserialize)] +pub struct CredentialsQuery { + /// Ledger alias (e.g., "mydb:main"). + pub ledger: String, +} + +/// GET /fluree/storage/credentials?ledger=... +/// +/// Mint STS credentials scoped to the ledger's S3 prefix so an authorized +/// peer reads index content directly from S3 (see +/// `docs/design/remote-mounts.md`). Guarded exactly like raw object serving: +/// bearer token scope, namespace guard, and the ledger's `f:serveBlocks` +/// posture — all failures answer 404 (no existence leak; consumers fall +/// back to proxied block reads). +/// +/// Requires `--storage-vend-enabled`, `--storage-vend-role-arn`, and +/// single-bucket S3-backed storage; otherwise the endpoint answers 404. +#[cfg(feature = "aws")] +pub async fn get_vended_credentials( + State(state): State>, + Query(query): Query, + StorageProxyBearer(principal): StorageProxyBearer, +) -> Result, ServerError> { + let (Some(scope), Some(role_arn)) = ( + state.storage_vend_scope.as_ref(), + state.config.storage_vend_role_arn.as_deref(), + ) else { + return Err(ServerError::not_found("Credential vending not available")); + }; + + // Namespace guard: `ledger` must be a real ledger (not a graph source). + let record = state + .fluree + .nameservice() + .lookup(&query.ledger) + .await + .map_err(|e| ServerError::internal(format!("Nameservice lookup failed: {e}")))? + .ok_or_else(|| ServerError::not_found("Ledger not found"))?; + let effective_ledger = record.ledger_id; + + // Authorize: token scope must include this ledger. + if !principal.is_authorized_for_ledger(&effective_ledger) { + return Err(ServerError::not_found("Ledger not found")); + } + + // Serving gate: the ledger's f:serveBlocks posture governs raw access. + // NOTE: unlike per-request block serving, a minted grant stays valid + // until it expires — posture changes take effect at grant expiry. + let serving = + crate::routes::serving::effective_serving(&state.fluree, &effective_ledger).await?; + if !serving.blocks { + return Err(ServerError::not_found("Ledger not found")); + } + + let grant = fluree_db_api::vended_credentials::mint_scoped_credentials_ambient( + role_arn, + scope, + &effective_ledger, + state.config.storage_vend_ttl_secs, + principal.identity.as_deref(), + ) + .await + .map_err(ServerError::Api)?; + + Ok(Json(grant)) +} + // ============================================================================ // Range Requests // ============================================================================ diff --git a/fluree-db-server/src/state.rs b/fluree-db-server/src/state.rs index b38a873831..a28b5e97d3 100644 --- a/fluree-db-server/src/state.rs +++ b/fluree-db-server/src/state.rs @@ -97,6 +97,12 @@ pub struct AppState { /// the presigned `.flpack` upload flow). Empty/unused unless /// `config.import_presign_enabled`. pub import_jobs: Arc, + + /// S3 vend scope for `GET /storage/credentials`, derived from the + /// connection config at startup. `None` when vending is disabled, the + /// backing storage isn't S3, or the commit/index buckets are split. + #[cfg(feature = "aws")] + pub storage_vend_scope: Option, } /// Spawn the periodic LeafletCache stats logger task. Returned @@ -219,6 +225,9 @@ impl AppState { index_config.clone(), )); + #[cfg(feature = "aws")] + let storage_vend_scope = resolve_storage_vend_scope(&config); + Ok(Self { fluree, config, @@ -237,6 +246,8 @@ impl AppState { query_refresh_last_checked: DashMap::new(), cache_stats_handle: Some(cache_stats_handle), import_jobs: Arc::new(crate::import_jobs::ImportJobs::default()), + #[cfg(feature = "aws")] + storage_vend_scope, }) } @@ -422,6 +433,66 @@ async fn build_direct_fluree( Ok((fluree, handle)) } +/// Derive the S3 vend scope from the server's connection config, when +/// credential vending is enabled and the backing storage is S3. +/// +/// Failures are downgraded to `None` with a log: the endpoint then answers +/// 404 (consumers fall back to proxied block reads) instead of the server +/// refusing to start. +#[cfg(feature = "aws")] +fn resolve_storage_vend_scope( + config: &ServerConfig, +) -> Option { + if !config.storage_vend_enabled { + return None; + } + if config.storage_vend_role_arn.is_none() { + tracing::error!( + "--storage-vend-enabled requires --storage-vend-role-arn; credential vending disabled" + ); + return None; + } + let Some(path) = &config.connection_config else { + tracing::error!( + "--storage-vend-enabled requires S3-backed storage (connection config); \ + credential vending disabled" + ); + return None; + }; + + let json_str = match std::fs::read_to_string(path) { + Ok(s) => s, + Err(e) => { + tracing::error!(error = %e, "vended credentials: cannot read connection config"); + return None; + } + }; + let json: serde_json::Value = match serde_json::from_str(&json_str) { + Ok(v) => v, + Err(e) => { + tracing::error!(error = %e, "vended credentials: cannot parse connection config"); + return None; + } + }; + let conn = match fluree_db_connection::ConnectionConfig::from_json_ld(&json) { + Ok(c) => c, + Err(e) => { + tracing::error!(error = %e, "vended credentials: invalid connection config"); + return None; + } + }; + + let scope = fluree_db_api::vended_credentials::S3VendScope::from_connection_config(&conn); + if scope.is_none() { + tracing::error!( + "--storage-vend-enabled requires single-bucket S3 storage; credential vending disabled" + ); + } else { + tracing::info!("vended S3 credentials enabled"); + } + scope +} + /// Build a proxy-backed `Fluree` for peer proxy mode. Reads land /// against a remote `ProxyNameService`; writes are forwarded to the /// transaction server. diff --git a/fluree-db-server/tests/proxy_integration.rs b/fluree-db-server/tests/proxy_integration.rs index 988ca2374b..e963869125 100644 --- a/fluree-db-server/tests/proxy_integration.rs +++ b/fluree-db-server/tests/proxy_integration.rs @@ -2384,6 +2384,37 @@ async fn test_serving_defaults_gate_and_advertisement() { ); } +/// Vended credentials answer 404 when vending is not configured — the +/// consumer's signal to fall back to proxied block reads. (Positive-path +/// minting is covered by the LocalStack test in fluree-db-api.) +#[cfg(feature = "aws")] +#[tokio::test] +async fn test_vended_credentials_404_when_unconfigured() { + let (_tmp, state) = tx_server_state().await; + let app = build_router(state); + + let secret = [0u8; 32]; + let signing_key = SigningKey::from_bytes(&secret); + let token = create_storage_proxy_token_no_identity(&signing_key, true); + + let resp = app + .oneshot( + Request::builder() + .method("GET") + .uri("/v1/fluree/storage/credentials?ledger=any:main") + .header("Authorization", format!("Bearer {token}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "file-backed server without vend config must answer 404" + ); +} + /// Discovery advertises the coarse server-level serving capabilities. #[tokio::test] async fn test_discovery_advertises_serving_capabilities() { From eff86453243ab907f2555144b41280381a4f0ed3 Mon Sep 17 00:00:00 2001 From: bplatz Date: Fri, 3 Jul 2026 16:46:36 -0400 Subject: [PATCH 6/9] docs: guide for sharing data with downstream consumers End-to-end guide covering the sharing patterns (query serving vs peer/block serving vs replication) with a decision table, provider setup (trusted issuers, token minting per tier, per-ledger f:servingDefaults participation, identity-bound row-level permissioning, vended S3 credentials), the consumer-side CLI workflow (remote add / auth login / remote ledgers / track modes / clone / cache), programmatic mounts, and the revocation/integrity/freshness semantics. Indexed in SUMMARY and the guides README. --- docs/SUMMARY.md | 1 + docs/guides/README.md | 4 + docs/guides/sharing-data.md | 304 ++++++++++++++++++++++++++++++++++++ 3 files changed, 309 insertions(+) create mode 100644 docs/guides/sharing-data.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 4762e54b72..090e06c79b 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -74,6 +74,7 @@ - [Time travel patterns](guides/cookbook-time-travel.md) - [Branching and merging](guides/cookbook-branching.md) - [Access control policies](guides/cookbook-policies.md) + - [Sharing data with downstream consumers](guides/sharing-data.md) - [SHACL validation](guides/cookbook-shacl.md) - [Edge annotations](guides/cookbook-edge-annotations.md) - [`owl:imports` across named graphs](guides/cookbook-owl-imports.md) diff --git a/docs/guides/README.md b/docs/guides/README.md index a04004896b..6bec2cc8a0 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -34,6 +34,10 @@ Git-like workflows for data: safe experimentation, review-before-merge, multi-en Set up fine-grained access control: department isolation, role-based access, property redaction, multi-tenant isolation, and default-deny patterns. +### [Sharing Data with Downstream Consumers](sharing-data.md) + +Serve your ledgers to other teams and organizations: choosing between query serving (your compute, row-level policy) and peer/block serving (their compute, whole-ledger), minting scoped tokens, declaring per-ledger participation with `f:servingDefaults`, identity-bound permissioning end to end, and the consumer-side CLI workflow. + ### [SHACL Validation](cookbook-shacl.md) Define data quality constraints: required properties, datatype validation, value ranges, string patterns, cardinality, and allowed values. diff --git a/docs/guides/sharing-data.md b/docs/guides/sharing-data.md new file mode 100644 index 0000000000..985fcf9176 --- /dev/null +++ b/docs/guides/sharing-data.md @@ -0,0 +1,304 @@ +# Sharing data with downstream consumers + +This guide is for a team running Fluree that wants other teams, partners, or +customers to consume its ledgers — and for those consumers. It covers the +sharing patterns, how a provider enables and scopes each one, how per-ledger +participation is declared, and what the consumer runs on their side. + +The one-paragraph model: a serving Fluree offers each ledger through up to +two tiers. With **query serving**, your server executes the consumer's +queries — your compute, with row-level policy applied per identity, so it's +the only tier that supports fine-grained permissioning. With **block +serving**, your server (or S3 directly) hands out the ledger's canonical +index content and the consumer queries it *locally* — their compute, like an +Iceberg catalog serving table files, but strictly all-or-nothing per ledger: +a consumer either may read the whole ledger or gets nothing. Deep dive: +[Remote mounts and serving tiers](../design/remote-mounts.md). + +## Choosing a pattern + +| | Query serving | Peer / block serving | Full replication | +|---|---|---|---| +| Whose compute | Provider's | Consumer's | Consumer's | +| Access granularity | Row-level (identity policy) | Whole ledger or nothing | Whole ledger or nothing | +| Provider cost per query | Query execution | Bandwidth on first touch, then ~nothing (CID cache) | None (one-time transfer) | +| Consumer freshness | Always current | Current (head check per query) | As of last `pull` | +| Consumer setup | Token only | Token + disk cache | Local storage for the full copy | +| CLI | `fluree track add` | `fluree track add --mode peer` | `fluree clone` / `fluree pull` | + +Rules of thumb: + +- **Fine-grained permissions → query serving.** It is the only pattern where + different identities see different rows of the same ledger. +- **Heavy or frequent analytical consumers you trust with the full ledger → + peer mode.** Your server stops paying query compute; with S3-backed + storage and [vended credentials](#optional-vended-s3-credentials) it stops + paying bandwidth too. +- **Offline or air-gapped consumers → replication** ([clone](../cli/clone.md), + [pull](../cli/pull.md)). +- A fourth mechanism, SPARQL `SERVICE ` federation, + composes remote query results *inside* your own queries — useful for + cross-organization joins, but it is query shipping under the hood and + follows the query-serving rules. + +The patterns compose per ledger and per consumer: the same server can serve +filtered queries to analysts and raw blocks to a partner's compute cluster. + +## Provider setup + +### 1. Keys and trusted issuers + +All sharing is authorized by Bearer tokens (JWS with an embedded Ed25519 +key, or OIDC — see [Authentication](../security/authentication.md)). Trust +is anchored in the **issuer**: your server only accepts tokens signed by +keys you configure. + +```bash +# One-time: generate a signing keypair. The did:key is the issuer. +fluree token keygen +# → private key + did:key:z6Mk... +``` + +Start the server trusting that issuer on the surfaces you intend to serve: + +```bash +fluree-server \ + --storage-path /var/lib/fluree \ + # query serving with enforced auth: + --data-auth-mode required \ + --data-auth-trusted-issuer did:key:z6Mk... \ + # block serving (peer/replication tier): + --storage-proxy-enabled \ + --storage-proxy-trusted-issuer did:key:z6Mk... +``` + +Operator details for each flag: [Query peers and +replication](../operations/query-peers.md). + +### 2. Mint consumer tokens + +The token's claims are the consumer's grant — which ledgers, which tier, +and (for query serving) which identity policy applies. See +[token](../cli/token.md) for the full surface. + +```bash +# Query-serving consumer: read two ledgers, policy-bound identity +fluree token create --private-key @signing.key \ + --identity "https://example.org/consumers/acme-analyst" \ + --read-ledger sales:main --read-ledger inventory:main \ + --expires-in 30d + +# Peer/block-serving consumer: full-access raw content for one ledger +fluree token create --private-key @signing.key \ + --storage-ledger analytics:main \ + --expires-in 30d +``` + +The claim vocabulary maps directly to tiers: `fluree.ledger.read.*` grants +query serving; `fluree.storage.*` grants block serving (and replication) — +**issue storage claims only to consumers entitled to the ledger's full +contents**, since raw index blocks bypass row-level policy by design. +`fluree.ledger.write.*` grants writes, which always execute on your server +regardless of how the consumer reads. + +Consumers only ever see what their token covers: the catalog endpoint +(`GET /nameservice/snapshot`, surfaced as `fluree remote ledgers`) filters +to the token's scope, and out-of-scope ledgers answer 404. + +### 3. Declare per-ledger participation + +By default every ledger is served on both tiers. To restrict a ledger, +transact the `f:servingDefaults` setting group into its config graph — it +travels with the ledger and needs no server restart (details: +[setting groups](../ledger-config/setting-groups.md)): + +```trig +@prefix f: . + +GRAPH { + a f:LedgerConfig ; + f:servingDefaults . + # "bring your own compute": blocks are served, queries are refused + f:serveQuery false ; + f:serveBlocks true . +} +``` + +- `f:serveQuery false` → query endpoints answer 403 for this ledger. +- `f:serveBlocks false` → the block/replication endpoints (`/storage/block`, + `/storage/objects`, `/commits`, `/pack`) answer 404; only query serving + remains. **This is the posture for any ledger with row-level policies** — + it forces all access through the policy-enforcing tier. +- The effective tiers are advertised per ledger, so consumers see what's on + offer (the `SERVING` column of `fluree remote ledgers`). + +These gates bind your server's serving surface. A consumer who already +holds the blocks (peer cache, clone) can always query their own copy — +that's inherent to handing over the data, not a policy gap. + +### 4. Row-level permissioning (query serving) + +Fine-grained sharing = query serving + identity-bound policy. Three pieces +must line up; the full policy language is in the +[policies cookbook](cookbook-policies.md). + +**a. Store the policies and tag the identity** (in the shared ledger). The +pattern is the cookbook's pair — a required restriction on the sensitive +property plus a default allow for everything else: + +```jsonld +{ + "ledger": "sales:main", + "insert": [ + { "@id": "ex:margin-restriction", "@type": ["f:AccessPolicy", "ex:PartnerClass"], + "f:required": true, + "f:onProperty": [{"@id": "ex:internalMargin"}], + "f:action": [{"@id": "f:view"}] }, + { "@id": "ex:partner-default-view", "@type": ["f:AccessPolicy", "ex:PartnerClass"], + "f:action": [{"@id": "f:view"}], + "f:allow": true }, + { "@id": "https://example.org/consumers/acme-analyst", + "f:policyClass": [{"@id": "ex:PartnerClass"}] } + ] +} +``` + +(The required policy has no `f:query`, so it never grants — +`ex:internalMargin` is simply invisible to this class. Conditional grants +and the full pattern library are in the cookbook.) + +**b. Mint the token with that identity** (`--identity` sets +`fluree.identity`, which the server binds non-spoofably to every request): + +```bash +fluree token create --private-key @signing.key \ + --identity "https://example.org/consumers/acme-analyst" \ + --read-ledger sales:main --expires-in 30d +``` + +**c. Keep the ledger query-only** (`f:serveBlocks false`, section 3), so +the policy tier is the only road in. + +The consumer does nothing special: they authenticate with the token, and +every query they send is evaluated as that identity — the policy classes +tagged on the identity load automatically. + +### 5. Optional: vended S3 credentials + +If your storage is S3, block-serving consumers can read **directly from +S3** instead of proxying bytes through your server — same access model, +near-zero serving cost: + +```bash +fluree-server ... \ + --storage-vend-enabled \ + --storage-vend-role-arn arn:aws:iam::123456789012:role/fluree-vend +``` + +Each grant is an STS session narrowed to the requested ledger's S3 prefix, +with a short TTL (default 15 minutes — grants outlive token revocation +until they expire, so keep TTLs short). Requirements and IAM notes: +[query-peers](../operations/query-peers.md). Consumers pick this up +automatically — no configuration on their side. + +## Consumer setup + +Everything below is driven from the provider's URL plus the token they +issued you. + +### Connect and discover + +```bash +fluree remote add acme https://data.acme.example +fluree auth login --remote acme --token @acme-token.jwt # or paste interactively + +fluree remote ledgers acme +# LEDGER COMMIT T INDEX T SERVING +# sales:main 1042 1040 query +# analytics:main 77 77 query+blocks +``` + +The `SERVING` column is your menu: `query` means the provider executes your +queries; `blocks` means you may run them locally in peer mode. + +### Pattern: provider executes your queries + +```bash +fluree track add sales --remote acme --remote-alias sales:main +fluree query sales 'SELECT ?product ?total WHERE { ... }' +``` + +Every query is an HTTP round-trip; results reflect the row-level policy +bound to your token's identity. Writes (if your token has write scope) work +the same way: `fluree insert sales ...`. + +### Pattern: your compute over their blocks (peer mode) + +```bash +fluree track add analytics --remote acme --remote-alias analytics:main --mode peer +fluree query analytics 'SELECT (COUNT(?s) AS ?n) WHERE { ?s a ex:Event }' +``` + +The first query streams the index blocks it touches (CID-verified) and +caches them on disk; subsequent queries mostly hit the cache, with one +cheap head check per query for freshness. If the provider vends S3 +credentials you'll see `notice: reading 'analytics:main' directly from S3`. +Writes still forward to the provider over HTTP — peer mode only changes +where *reads* execute. Cache maintenance: `fluree cache status` / +`fluree cache clear` (always safe — everything is content-addressed and +re-fetched on demand). See [track](../cli/track.md) and +[cache](../cli/cache.md). + +### Pattern: full replication + +```bash +fluree clone acme analytics # full copy into local storage +fluree pull analytics # catch up later +``` + +After a clone the ledger is fully local — queryable offline, no further +contact with the provider until you pull. Requires the same storage-scope +token as peer mode. + +### Programmatic (Rust) consumers + +An application embedding Fluree mounts a remote next to its own ledgers — +mounted aliases are read-only and mix freely with local ledgers in one +query: + +```rust +let storage = ProxyStorage::from_api_base(url.clone(), token.clone(), ProxyReadMode::Raw) + .with_local_prefix("acme"); +let ns = Arc::new(ProxyNameService::from_api_base(url, token)); +let fluree = FlureeBuilder::memory() + .with_remote_mount(RemoteMountSpec::new("acme", ns, storage)) + .build_memory(); +// fluree.graph("acme/analytics:main").query()… — local compute, remote bytes +// FROM ["books:main", "acme/analytics:main"] federates local + mounted +``` + +## Semantics worth knowing + +- **Revocation.** Query serving revokes at token expiry or issuer removal, + per request. Block serving is revoked for *new* fetches the same way, but + blocks a consumer already fetched (and any vended S3 grant) remain usable + until cache-cleared / expired — all-or-nothing sharing means the handed-over + data is theirs. Posture changes (`f:servingDefaults`) apply on the next + request. +- **Integrity.** Everything on the block tier is content-addressed and + verified against its CID on both ends; a consumer's cache never goes + stale (only nameservice heads move) and is always safe to delete. +- **Freshness.** Peer-mode queries check the provider's head each query, so + reads are as current as the provider's index. Time travel works against + whatever history the provider retains. +- **Don't mix tiers for permissioned data.** A ledger with row-level + policies must not grant `fluree.storage.*` tokens or `f:serveBlocks` — + raw blocks carry every row. + +## Related documentation + +- [Remote mounts and serving tiers](../design/remote-mounts.md) — the design +- [Query peers and replication](../operations/query-peers.md) — operator reference +- [Policies cookbook](cookbook-policies.md) — the policy language +- [Authentication](../security/authentication.md) · [Auth contract](../design/auth-contract.md) +- CLI: [remote](../cli/remote.md) · [track](../cli/track.md) · [cache](../cli/cache.md) · [token](../cli/token.md) · [clone](../cli/clone.md) · [pull](../cli/pull.md) From 929b0249759fc5bd374f331f1c89a0f58b8b43d1 Mon Sep 17 00:00:00 2001 From: bplatz Date: Fri, 3 Jul 2026 17:20:53 -0400 Subject: [PATCH 7/9] docs: server-integration contract for peer-mode block serving Document the endpoints an embedding server must expose for CLI peer mode (fluree track --mode peer) and fluree remote ledgers: the NsRecord lookup and CAS object endpoints with their required semantics (all-or-nothing authorization with 404 anti-leak, dict-blob branch resolution for name-scoped @shared artifacts, exact CID-verifiable bytes), recommended Range support, and the optional vended-credentials endpoint with its 404-fallback contract. --- docs/cli/server-integration.md | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/docs/cli/server-integration.md b/docs/cli/server-integration.md index 0d07675e47..435d07a7e4 100644 --- a/docs/cli/server-integration.md +++ b/docs/cli/server-integration.md @@ -50,6 +50,73 @@ Fallbacks (strongly recommended): - `GET {api_base_url}/commits/*ledger` (paginated export of commit + txn blobs) - `GET {api_base_url}/storage/objects/:cid?ledger=:ledger-id` (per-object fetch by CID) +### `fluree track add --mode peer` (local query execution over served blocks) + +Peer mode runs the consumer's queries **locally** over index blocks fetched +on demand from your server (see +[Sharing data with downstream consumers](../guides/sharing-data.md)). It is +read-path only — writes and every other command on a peer-tracked ledger +use the normal data/transaction endpoints above. + +Required: + +- `GET {api_base_url}/storage/ns/:ledger-id` — NsRecord JSON. Peer mode + reads `commit_head_id`, `index_head_id`, `commit_t`/`index_t` (head + freshness is checked per query), `source_branch`/`branches` (branched + ledgers), and the optional `serving` array (`["query","blocks"]`) that + advertises the tiers on offer. Unknown fields are ignored, so additions + are safe. +- `GET {api_base_url}/storage/objects/:cid?ledger=:ledger-id` — canonical + CAS bytes for all replication-relevant kinds **including raw index leaves + and dictionary blobs**. The client verifies every payload against its CID, + so bytes must be exact. + +Required semantics: + +- **All-or-nothing authorization.** These endpoints serve a ledger's full + contents with no row-level filtering, so a token must only be honored for + ledgers the bearer may read in full (`fluree.storage.all` / + `fluree.storage.ledgers` claims in the reference implementation). Return + **404** for out-of-scope, unknown, or serving-disabled ledgers — never + 403 (no existence leak; the CLI treats 404 as not-found). +- **Dict-blob branch resolution.** Shared dictionary artifacts live under a + name-scoped `@shared` namespace, so the client cannot know the branch and + derives a **default-branch** alias (e.g. `mydb:main`) even when the ledger + only exists on another branch. For dict-blob CIDs, resolve the `ledger` + parameter to any live branch of the same name before authorizing and + reading — otherwise peers tracking non-default branches fail on + dictionary fetches. + +Strongly recommended: + +- **Single-range `Range: bytes=start-end` support** on `/storage/objects` + (`206` + `Content-Range`, `416` past-end), verifying the full object + against the CID before slicing. Without it the client falls back to + whole-object fetches for leaflet-granular reads. + +Optional: + +- `GET {api_base_url}/storage/credentials?ledger=:ledger-id` — vended S3 + credentials so peers read directly from S3 instead of proxying bytes + through your server. The CLI probes this once per peer query invocation + and **falls back to `/storage/objects` on 404**, so serving 404 is a + complete, valid implementation. If you do implement it, return the grant + shape documented in [endpoints](../api/endpoints.md#get-storagecredentials) + (`access_key_id`, `secret_access_key`, `session_token`, + `expires_at_epoch_secs`, `bucket`, `region`, optional `endpoint` / + `key_prefix`) with credentials scoped to the requested ledger's storage + prefix; the client auto-refreshes as grants approach expiry. + +### `fluree remote ledgers` (auth-filtered catalog) + +- `GET {api_base_url}/nameservice/snapshot` — `{ "ledgers": [NsRecord…], + "graph_sources": […] }`, filtered to the presented token's scope (an + unauthorized caller sees an empty or partial list, not an error). +- `GET {api_base_url}/storage/ns/:ledger-id` — fetched per ledger to fill + the `SERVING` column from the record's `serving` array; the column + renders `-` when the field or record is unavailable, so this + degrades gracefully. + ### `fluree push` (commit ingestion) - `POST {api_base_url}/push/*ledger` From d1034aa0391a4747162e429b3942c4fc2203ed4f Mon Sep 17 00:00:00 2001 From: bplatz Date: Sun, 5 Jul 2026 09:57:07 -0400 Subject: [PATCH 8/9] perf(serving): targeted serving-config resolver + per-(ledger,t) posture memo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The serving gate resolved the entire ledger config graph (all 8 setting groups + a type scan) just to read f:servingDefaults, on every query to an origin server and on every object of a peer cold sync. - config_resolver: extract the shared config-subject lookup into resolve_config_sid and add resolve_serving_only, which reads only the serving group. effective_serving_from_state now uses it, cutting 7 unused group reads at every serving gate. - AppState::effective_serving_cached memoizes EffectiveServing per (ledger, t) so the peer raw-read path (get_object_by_cid, one call per index leaf/branch/ root/dict/commit) resolves the posture once per head instead of per object. Keyed on the novelty-inclusive t (state.t()), so a config change — which lands in novelty and bumps t before any reindex — invalidates the entry; one entry per ledger, so the map stays bounded. Replaces the free effective_serving fn at its three storage_proxy call sites. --- fluree-db-api/src/config_resolver.rs | 75 ++++++++++++++------ fluree-db-server/src/routes/serving.rs | 26 ++----- fluree-db-server/src/routes/storage_proxy.rs | 8 +-- fluree-db-server/src/state.rs | 43 +++++++++++ 4 files changed, 104 insertions(+), 48 deletions(-) diff --git a/fluree-db-api/src/config_resolver.rs b/fluree-db-api/src/config_resolver.rs index 57c98e5976..0ef3df3f4a 100644 --- a/fluree-db-api/src/config_resolver.rs +++ b/fluree-db-api/src/config_resolver.rs @@ -55,6 +55,44 @@ pub async fn resolve_ledger_config( overlay: &dyn OverlayProvider, to_t: i64, ) -> Result> { + let Some(config_sid) = resolve_config_sid(snapshot, overlay, to_t).await? else { + return Ok(None); + }; + + // Read the config_id (@id) + let config_id = snapshot.decode_sid(&config_sid); + + // Read each setting group + let policy = read_policy_defaults(snapshot, overlay, to_t, &config_sid).await?; + let shacl = read_shacl_defaults(snapshot, overlay, to_t, &config_sid).await?; + let reasoning = read_reasoning_defaults(snapshot, overlay, to_t, &config_sid).await?; + let datalog = read_datalog_defaults(snapshot, overlay, to_t, &config_sid).await?; + let transact = read_transact_defaults(snapshot, overlay, to_t, &config_sid).await?; + let full_text = read_fulltext_defaults(snapshot, overlay, to_t, &config_sid).await?; + let serving = read_serving_defaults(snapshot, overlay, to_t, &config_sid).await?; + let graph_overrides = read_graph_overrides(snapshot, overlay, to_t, &config_sid).await?; + + Ok(Some(LedgerConfig { + config_id, + policy, + shacl, + reasoning, + datalog, + transact, + full_text, + serving, + graph_overrides, + })) +} + +/// Locate the single `f:LedgerConfig` subject in the config graph as-of `to_t`, +/// or `None` when the ledger has no config. Shared prefix of +/// [`resolve_ledger_config`] and [`resolve_serving_only`]. +async fn resolve_config_sid( + snapshot: &LedgerSnapshot, + overlay: &dyn OverlayProvider, + to_t: i64, +) -> Result> { // Cheap guard: if the config graph (CONFIG_GRAPH_ID) holds no data in either // the novelty overlay or the base index, there can be no `f:LedgerConfig` — // skip the type scan entirely. Without this, the `?s rdf:type f:LedgerConfig` @@ -129,30 +167,21 @@ pub async fn resolve_ledger_config( with_iris.into_iter().next().unwrap().1 }; - // Read the config_id (@id) - let config_id = snapshot.decode_sid(&config_sid); - - // Read each setting group - let policy = read_policy_defaults(snapshot, overlay, to_t, &config_sid).await?; - let shacl = read_shacl_defaults(snapshot, overlay, to_t, &config_sid).await?; - let reasoning = read_reasoning_defaults(snapshot, overlay, to_t, &config_sid).await?; - let datalog = read_datalog_defaults(snapshot, overlay, to_t, &config_sid).await?; - let transact = read_transact_defaults(snapshot, overlay, to_t, &config_sid).await?; - let full_text = read_fulltext_defaults(snapshot, overlay, to_t, &config_sid).await?; - let serving = read_serving_defaults(snapshot, overlay, to_t, &config_sid).await?; - let graph_overrides = read_graph_overrides(snapshot, overlay, to_t, &config_sid).await?; + Ok(Some(config_sid)) +} - Ok(Some(LedgerConfig { - config_id, - policy, - shacl, - reasoning, - datalog, - transact, - full_text, - serving, - graph_overrides, - })) +/// Resolve only the serving posture (`f:servingDefaults`), skipping the other +/// seven setting groups. The serving gates read only `config.serving`, so this +/// avoids the wasted group reads a full [`resolve_ledger_config`] would do. +pub async fn resolve_serving_only( + snapshot: &LedgerSnapshot, + overlay: &dyn OverlayProvider, + to_t: i64, +) -> Result> { + let Some(config_sid) = resolve_config_sid(snapshot, overlay, to_t).await? else { + return Ok(None); + }; + read_serving_defaults(snapshot, overlay, to_t, &config_sid).await } // ============================================================================ diff --git a/fluree-db-server/src/routes/serving.rs b/fluree-db-server/src/routes/serving.rs index 09a35953fc..549ab7ed06 100644 --- a/fluree-db-server/src/routes/serving.rs +++ b/fluree-db-server/src/routes/serving.rs @@ -12,7 +12,7 @@ //! access tier. use crate::error::ServerError; -use fluree_db_api::{config_resolver, Fluree, LedgerState}; +use fluree_db_api::{config_resolver, LedgerState}; use fluree_db_core::ledger_config::ServingDefaults; use fluree_db_core::OverlayProvider; @@ -60,26 +60,12 @@ pub(crate) async fn effective_serving_from_state( state: &LedgerState, ) -> Result { let overlay: &dyn OverlayProvider = &*state.novelty; - let config = config_resolver::resolve_ledger_config(&state.snapshot, overlay, state.t()) + let serving = config_resolver::resolve_serving_only(&state.snapshot, overlay, state.t()) .await .map_err(|e| ServerError::internal(format!("Serving config resolution failed: {e}")))?; - Ok(EffectiveServing::from_config( - config.as_ref().and_then(|c| c.serving.as_ref()), - )) + Ok(EffectiveServing::from_config(serving.as_ref())) } -/// Resolve the serving posture for a ledger by alias. -/// -/// Loads (or reuses) the cached ledger handle. Callers on anti-leak paths -/// should map load failures to their endpoint's 404 convention. -pub(crate) async fn effective_serving( - fluree: &Fluree, - ledger_id: &str, -) -> Result { - let handle = fluree - .ledger_cached(ledger_id) - .await - .map_err(ServerError::Api)?; - let state = handle.snapshot().await.to_ledger_state(); - effective_serving_from_state(&state).await -} +// Resolving serving posture by ledger alias goes through +// `AppState::effective_serving_cached`, which memoizes the posture per +// `(ledger, t)` — see fluree-db-server/src/state.rs. diff --git a/fluree-db-server/src/routes/storage_proxy.rs b/fluree-db-server/src/routes/storage_proxy.rs index 98ec6252ba..fccc85e4c2 100644 --- a/fluree-db-server/src/routes/storage_proxy.rs +++ b/fluree-db-server/src/routes/storage_proxy.rs @@ -284,8 +284,7 @@ pub async fn get_vended_credentials( // Serving gate: the ledger's f:serveBlocks posture governs raw access. // NOTE: unlike per-request block serving, a minted grant stays valid // until it expires — posture changes take effect at grant expiry. - let serving = - crate::routes::serving::effective_serving(&state.fluree, &effective_ledger).await?; + let serving = state.effective_serving_cached(&effective_ledger).await?; if !serving.blocks { return Err(ServerError::not_found("Ledger not found")); } @@ -409,7 +408,7 @@ pub async fn get_ns_record( // Advertise the serving tiers this server offers for the ledger. // Best-effort: a ledger that fails to load can't resolve its posture, // and the record lookup itself should still succeed. - let serving = match crate::routes::serving::effective_serving(&state.fluree, &ledger_id).await { + let serving = match state.effective_serving_cached(&ledger_id).await { Ok(s) => Some(s.advertised()), Err(e) => { tracing::warn!(ledger_id, error = %e, "serving posture unresolved; omitting from NS record"); @@ -716,8 +715,7 @@ pub async fn get_object_by_cid( // 3c. Serving gate: the ledger's f:serveBlocks posture must allow raw // content serving. - let serving = - crate::routes::serving::effective_serving(&state.fluree, &effective_ledger).await?; + let serving = state.effective_serving_cached(&effective_ledger).await?; if !serving.blocks { return Err(ServerError::not_found("Object not found")); } diff --git a/fluree-db-server/src/state.rs b/fluree-db-server/src/state.rs index a28b5e97d3..9661979417 100644 --- a/fluree-db-server/src/state.rs +++ b/fluree-db-server/src/state.rs @@ -89,6 +89,15 @@ pub struct AppState { /// from stampeding the nameservice. query_refresh_last_checked: DashMap, + /// Serving posture memo, keyed by ledger id with value `(t, posture)`. + /// + /// The peer raw-read path resolves the serving gate per object; without + /// this, a cold sync re-resolves the config graph for every leaf/branch/ + /// root/dict/commit at the same `t`. One entry per ledger, overwritten when + /// `t` advances (a config change bumps `t`), so it stays bounded and + /// self-invalidating. + serving_posture_cache: DashMap, + /// Handle for the background leaflet cache stats logger task. /// Aborted on drop so the `Arc` doesn't outlive the server. cache_stats_handle: Option>, @@ -143,6 +152,39 @@ fn spawn_leaflet_cache_stats_logger(fluree: &Arc) -> tokio::task::JoinHa } impl AppState { + /// Serving posture for a ledger, memoized per `(ledger, t)`. + /// + /// Resolves the serving gate for the ledger's current head, reusing a + /// cached posture when nothing has committed since (same `t`). The cheap + /// work (cached handle + ledger state) still runs each call; the memo skips + /// the config-graph resolution — the actual per-object cost on the peer + /// raw-read path. See [`Self::serving_posture_cache`]. + pub(crate) async fn effective_serving_cached( + &self, + ledger_id: &str, + ) -> Result { + let handle = self + .fluree + .ledger_cached(ledger_id) + .await + .map_err(crate::error::ServerError::Api)?; + // Key on the novelty-inclusive `t` (what the resolver reads via + // `state.t()`), NOT the indexed base `snapshot.t`: a config change lands + // in novelty and bumps the effective `t` without touching the base until + // a reindex, so keying on the base would serve a stale posture. + let ledger_state = handle.snapshot().await.to_ledger_state(); + let t = ledger_state.t(); + if let Some(entry) = self.serving_posture_cache.get(ledger_id) { + if entry.0 == t { + return Ok(entry.1); + } + } + let serving = crate::routes::serving::effective_serving_from_state(&ledger_state).await?; + self.serving_posture_cache + .insert(ledger_id.to_string(), (t, serving)); + Ok(serving) + } + /// Create new application state from config. /// /// Convenience shortcut: builds the default `Fluree` instance @@ -244,6 +286,7 @@ impl AppState { raft: None, refresh_counter: AtomicU64::new(0), query_refresh_last_checked: DashMap::new(), + serving_posture_cache: DashMap::new(), cache_stats_handle: Some(cache_stats_handle), import_jobs: Arc::new(crate::import_jobs::ImportJobs::default()), #[cfg(feature = "aws")] From 8a58a984168ab6b9c679b39bd5fc1e751b01882d Mon Sep 17 00:00:00 2001 From: bplatz Date: Sun, 5 Jul 2026 10:01:23 -0400 Subject: [PATCH 9/9] memory --- .fluree-memory/repo.ttl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.fluree-memory/repo.ttl b/.fluree-memory/repo.ttl index b29fd8b8fc..fbc3a8e430 100644 --- a/.fluree-memory/repo.ttl +++ b/.fluree-memory/repo.ttl @@ -1567,6 +1567,21 @@ mem:fact-01kwmb5yq11y50j3kbd7c5tne4 a mem:Fact ; mem:branch "feature/remote-mounts" ; mem:createdAt "2026-07-03T15:57:26.113125+00:00"^^xsd:dateTime . +mem:fact-01kws95166gqpjjfda0nvmkraq a mem:Fact ; + mem:content "Any per-(ledger,t) cache of config-derived server state MUST key on the novelty-inclusive t (LedgerState::t() / state.t()), NOT the indexed base snapshot.snapshot.t. A config-graph change (e.g. f:servingDefaults) commits into novelty and bumps the effective t WITHOUT advancing the indexed base t until a reindex. Keying on the base t serves a stale posture across config changes. This exact bug hit AppState::effective_serving_cached and was caught by test_serving_defaults_gate_and_advertisement (proxy_integration.rs): got 200 expecting 404 because the cached entry didn't invalidate. resolve_ledger_config / effective_serving_from_state resolve at state.t(), so the cache key must match." ; + mem:tag "cache" ; + mem:tag "config-graph" ; + mem:tag "novelty" ; + mem:tag "server" ; + mem:tag "serving" ; + mem:scope mem:repo ; + mem:artifactRef "fluree-db-api/src/config_resolver.rs" ; + mem:artifactRef "fluree-db-server/src/routes/serving.rs" ; + mem:artifactRef "fluree-db-server/src/state.rs" ; + mem:branch "feature/remote-mounts" ; + mem:createdAt "2026-07-05T13:58:10.886197+00:00"^^xsd:dateTime ; + mem:rationale "Non-obvious: indexed snapshot t and effective (novelty-inclusive) t diverge between a commit and the next reindex; config changes live in novelty, so t-keyed caches of config-derived state silently go stale if keyed on the base t." . + mem:fact-01kvzn4mmmcvc7zz6bcswtw47c a mem:Fact ; mem:content "SPARQL property paths, branch feature/sparql-property-paths: closed ALL reported gaps PLUS inverse-in-composite (wired in BOTH sparql lower/path.rs AND query parse/lower.rs for JSON-LD parity). (1) ZeroOrOne p?: PathModifier::ZeroOrOne; BFS self+1hop. (2) Negated sets !: fresh ?__np{n} pred-var triple + FILTER(?p != IRI(...)); fwd/inv/mixed=UNION. (3) Nested modifiers (p*)*: collapse_modifiers (zero=any */?, unbounded=any +/*). (4) Composite-transitive (a/b)+ incl inverse steps (^a/b)+: PropertyPathPattern has predicates+first_inverse+sequence_steps:Vec; operator's read_step reads SPOT/POST by direction (opposite index when retreating); single_predicate() bails on composite. ONLY UNSUPPORTED (niche): nested transitive step in a composite unit (a+/b)+ — not exercised by any W3C test." ; mem:tag "negated-set" ;