From 2fc21707291f06c7d28b6a8f37d23548c712823e Mon Sep 17 00:00:00 2001 From: Christophe Di Prima Date: Fri, 26 Jun 2026 16:35:29 +0200 Subject: [PATCH 1/5] feat(iceberg): append-only incremental scan + snapshot-window helpers Adds the read half of incremental materialization: - SendScanPlanner::plan_incremental(from, to): the data files ADDED in a (from, to] sequence-number window (live entries whose effective data sequence number > from.sequence_number), reusing the existing manifest walk. - SendScanPlanner::plan_scan_with_selection: full scan at a chosen snapshot. - effective_sequence_number(): Iceberg null/0 seq inheritance from the manifest. - TableMetadata::snapshot_window / window_is_append_only: parent-chain ancestry walk + append-only detection, so the materialize layer can fall back to a full re-read when the window has overwrite/delete/replace (updates/deletes the added-files scan can't see) or from is not an ancestor of to. Unit tests cover seq-number inheritance, the ancestry walk (incl. branch/expired errors), and append-only detection. Row-level delete-file (v2 MoR) support is future work. Co-Authored-By: Claude Opus 4.8 (1M context) --- fluree-db-iceberg/src/metadata/table.rs | 161 +++++++++++++++++++++ fluree-db-iceberg/src/scan/planner.rs | 56 +++++++ fluree-db-iceberg/src/scan/send_planner.rs | 131 ++++++++++++++++- 3 files changed, 347 insertions(+), 1 deletion(-) diff --git a/fluree-db-iceberg/src/metadata/table.rs b/fluree-db-iceberg/src/metadata/table.rs index 1ca8931503..04e35744d5 100644 --- a/fluree-db-iceberg/src/metadata/table.rs +++ b/fluree-db-iceberg/src/metadata/table.rs @@ -107,6 +107,65 @@ impl TableMetadata { pub fn default_partition_spec(&self) -> Option<&PartitionSpec> { self.partition_spec(self.default_spec_id) } + + /// The snapshots in the window `(from_id, to_id]`, newest first, walking the + /// `parent_snapshot_id` chain from `to_id` back toward `from_id`. + /// + /// `from_id = None` walks to the root (the full history up to `to_id`). + /// Returns an error if `to_id` is unknown, an ancestor is missing (e.g. an + /// expired snapshot), or `from_id` is not an ancestor of `to_id` (a branch or + /// rollback) — in all of which the caller should fall back to a full re-read. + pub fn snapshot_window( + &self, + from_id: Option, + to_id: i64, + ) -> crate::error::Result> { + if from_id == Some(to_id) { + return Ok(Vec::new()); + } + let mut window = Vec::new(); + let mut cur = self.snapshot(to_id).ok_or_else(|| { + crate::error::IcebergError::SnapshotNotFound(format!("snapshot {to_id} not found")) + })?; + loop { + window.push(cur); + match cur.parent_snapshot_id { + Some(pid) if Some(pid) == from_id => return Ok(window), + Some(pid) => { + cur = self.snapshot(pid).ok_or_else(|| { + crate::error::IcebergError::SnapshotNotFound(format!( + "ancestor snapshot {pid} not found (history may be expired)" + )) + })?; + } + None => { + // Reached the root snapshot. + if from_id.is_none() { + return Ok(window); + } + return Err(crate::error::IcebergError::Metadata(format!( + "snapshot {} is not an ancestor of {to_id}", + from_id.unwrap() + ))); + } + } + } + } + + /// Whether every snapshot in `(from_id, to_id]` was created by an `append` + /// operation. Only then does an added-files incremental scan capture all + /// changes (no `overwrite`/`delete`/`replace` => no updates or deletions to + /// miss). A snapshot with no recorded operation is treated as not-append-only + /// (fail safe: caller should full-refresh). Propagates `snapshot_window` + /// errors (unknown/expired/non-ancestor). + pub fn window_is_append_only( + &self, + from_id: Option, + to_id: i64, + ) -> crate::error::Result { + let window = self.snapshot_window(from_id, to_id)?; + Ok(window.iter().all(|s| s.operation() == Some("append"))) + } } /// Schema definition. @@ -331,4 +390,106 @@ mod tests { let metadata = TableMetadata::from_json_str(SAMPLE_METADATA).unwrap(); assert_eq!(metadata.properties.get("owner"), Some(&"test".to_string())); } + + // ---- incremental window helpers ---- + + fn snap(id: i64, parent: Option, seq: i64, op: Option<&str>) -> crate::metadata::Snapshot { + let mut summary = HashMap::new(); + if let Some(o) = op { + summary.insert("operation".to_string(), o.to_string()); + } + crate::metadata::Snapshot { + snapshot_id: id, + parent_snapshot_id: parent, + sequence_number: seq, + timestamp_ms: seq * 1000, + manifest_list: Some(format!("snap-{id}.avro")), + manifests: None, + summary, + schema_id: Some(0), + } + } + + fn meta_with(snapshots: Vec) -> TableMetadata { + let current = snapshots.last().map(|s| s.snapshot_id); + let last_seq = snapshots + .iter() + .map(|s| s.sequence_number) + .max() + .unwrap_or(0); + TableMetadata { + format_version: 2, + table_uuid: None, + location: "s3://b/t".to_string(), + last_sequence_number: last_seq, + last_updated_ms: 0, + last_column_id: 1, + current_schema_id: 0, + schemas: vec![], + current_snapshot_id: current, + snapshots, + snapshot_log: vec![], + default_spec_id: 0, + partition_specs: vec![], + last_partition_id: 0, + sort_orders: vec![], + default_sort_order_id: 0, + properties: HashMap::new(), + } + } + + #[test] + fn snapshot_window_walks_parent_chain() { + let m = meta_with(vec![ + snap(1, None, 1, Some("append")), + snap(2, Some(1), 2, Some("append")), + snap(3, Some(2), 3, Some("append")), + ]); + // (1, 3] -> snapshots 3 then 2 (newest first), excluding the `from` (1) + let ids: Vec = m + .snapshot_window(Some(1), 3) + .unwrap() + .iter() + .map(|s| s.snapshot_id) + .collect(); + assert_eq!(ids, vec![3, 2]); + // from == to -> empty + assert!(m.snapshot_window(Some(3), 3).unwrap().is_empty()); + // from = None -> full history + let full: Vec = m + .snapshot_window(None, 3) + .unwrap() + .iter() + .map(|s| s.snapshot_id) + .collect(); + assert_eq!(full, vec![3, 2, 1]); + // `from` not an ancestor of `to` (branch/rollback) -> error + assert!(m.snapshot_window(Some(99), 3).is_err()); + // unknown `to` -> error + assert!(m.snapshot_window(Some(1), 42).is_err()); + } + + #[test] + fn window_is_append_only_detects_non_append() { + let all_append = meta_with(vec![ + snap(1, None, 1, Some("append")), + snap(2, Some(1), 2, Some("append")), + ]); + assert!(all_append.window_is_append_only(Some(1), 2).unwrap()); + // from == to -> empty window -> vacuously append-only (nothing to apply) + assert!(all_append.window_is_append_only(Some(2), 2).unwrap()); + + let with_overwrite = meta_with(vec![ + snap(1, None, 1, Some("append")), + snap(2, Some(1), 2, Some("overwrite")), + ]); + assert!(!with_overwrite.window_is_append_only(Some(1), 2).unwrap()); + + // A snapshot with no recorded operation is not provably append-only. + let no_op = meta_with(vec![ + snap(1, None, 1, Some("append")), + snap(2, Some(1), 2, None), + ]); + assert!(!no_op.window_is_append_only(Some(1), 2).unwrap()); + } } diff --git a/fluree-db-iceberg/src/scan/planner.rs b/fluree-db-iceberg/src/scan/planner.rs index 85ec1b8ad9..addb97b293 100644 --- a/fluree-db-iceberg/src/scan/planner.rs +++ b/fluree-db-iceberg/src/scan/planner.rs @@ -159,6 +159,51 @@ impl ScanPlan { } } +/// An incremental (append-only) scan plan: the data-file tasks ADDED in a +/// `(from_sequence_number, to]` window, used to materialize only the rows that +/// appeared since the last sync. +/// +/// This captures the **added** half of a changelog only; callers must ensure the +/// window contains no `overwrite`/`delete`/`replace` snapshots (see +/// [`TableMetadata::window_is_append_only`](crate::metadata::TableMetadata::window_is_append_only)) +/// and otherwise fall back to a full re-materialization. +#[derive(Debug)] +pub struct IncrementalScanPlan { + /// Tasks for data files added in the window. + pub added_tasks: Vec, + /// Projected column names (for reference). + pub projected_columns: Vec, + /// Projected field IDs. + pub projected_field_ids: Vec, + /// Exclusive lower bound: only files with effective data sequence number + /// strictly greater than this are included (`0` => from genesis). + pub from_sequence_number: i64, + /// The snapshot the window ends at (inclusive). + pub to_snapshot_id: i64, + /// Sequence number of `to_snapshot_id`. + pub to_sequence_number: i64, + /// Number of added files selected. + pub files_selected: usize, + /// Estimated added row count. + pub estimated_row_count: i64, +} + +/// Effective data sequence number of a manifest entry, applying Iceberg's +/// inheritance rule: an entry whose own sequence number is null/`0` inherits the +/// sequence number of the manifest (i.e. the manifest-list entry) that holds it. +/// +/// This is what makes `ADDED` entries (often written with a null sequence number) +/// resolve to the sequence number of the snapshot that added them. +pub fn effective_sequence_number( + entry_sequence_number: Option, + manifest_sequence_number: i64, +) -> i64 { + match entry_sequence_number { + Some(s) if s != 0 => s, + _ => manifest_sequence_number, + } +} + /// Scan planner for Iceberg tables. pub struct ScanPlanner<'a, S: IcebergStorage> { storage: &'a S, @@ -363,4 +408,15 @@ mod tests { assert_eq!(task.start, 0); assert_eq!(task.length, 10240); } + + #[test] + fn test_effective_sequence_number_inheritance() { + // An explicit, non-zero entry sequence number is used as-is. + assert_eq!(effective_sequence_number(Some(7), 3), 7); + // A null sequence number inherits the manifest's (ADDED entries are + // commonly written with a null sequence number). + assert_eq!(effective_sequence_number(None, 5), 5); + // Zero is treated as null and also inherits. + assert_eq!(effective_sequence_number(Some(0), 5), 5); + } } diff --git a/fluree-db-iceberg/src/scan/send_planner.rs b/fluree-db-iceberg/src/scan/send_planner.rs index 81840860a2..a36907e73f 100644 --- a/fluree-db-iceberg/src/scan/send_planner.rs +++ b/fluree-db-iceberg/src/scan/send_planner.rs @@ -9,7 +9,9 @@ use crate::error::{IcebergError, Result}; use crate::io::SendIcebergStorage; use crate::manifest::{parse_manifest, parse_manifest_list}; use crate::metadata::{Schema, Snapshot, TableMetadata}; -use crate::scan::planner::{FileScanTask, ScanConfig, ScanPlan}; +use crate::scan::planner::{ + effective_sequence_number, FileScanTask, IncrementalScanPlan, ScanConfig, ScanPlan, +}; use crate::scan::pruning::can_contain_file; /// Send-safe scan planner for Iceberg tables. @@ -130,6 +132,133 @@ impl<'a, S: SendIcebergStorage> SendScanPlanner<'a, S> { }) } + /// Plan a full scan for a snapshot chosen by `selection` (current / by id / + /// as-of-time) rather than always the current snapshot. + pub async fn plan_scan_with_selection( + &self, + selection: &crate::metadata::SnapshotSelection, + ) -> Result { + let snapshot = + crate::metadata::select_snapshot(self.metadata, selection).ok_or_else(|| { + IcebergError::SnapshotNotFound("No snapshot matches selection".to_string()) + })?; + self.plan_scan_for_snapshot(snapshot).await + } + + /// Plan an **incremental (append-only)** scan: the data files ADDED in the + /// window `(from_snapshot_id, to_snapshot_id]`. + /// + /// A file is "added in the window" when its *effective data sequence number* + /// (see [`effective_sequence_number`]) is strictly greater than the `from` + /// snapshot's sequence number. `from_snapshot_id = None` means "since genesis" + /// (the full live state of `to`) — used for the initial materialization. + /// + /// This captures **additions only**. The caller MUST first verify the window + /// is append-only via [`TableMetadata::window_is_append_only`](crate::metadata::TableMetadata::window_is_append_only) + /// and fall back to a full re-read otherwise — overwrite/delete/replace + /// snapshots carry updates/deletions this scan cannot see. + pub async fn plan_incremental( + &self, + from_snapshot_id: Option, + to_snapshot_id: i64, + ) -> Result { + let to_snapshot = self.metadata.snapshot(to_snapshot_id).ok_or_else(|| { + IcebergError::SnapshotNotFound(format!("to snapshot {to_snapshot_id} not found")) + })?; + let from_seq = match from_snapshot_id { + Some(id) => { + self.metadata + .snapshot(id) + .ok_or_else(|| { + IcebergError::SnapshotNotFound(format!("from snapshot {id} not found")) + })? + .sequence_number + } + None => 0, + }; + let to_seq = to_snapshot.sequence_number; + + let schema = self + .metadata + .current_schema() + .ok_or_else(|| IcebergError::Metadata("No current schema".to_string()))?; + let schema_arc = Arc::new(schema.clone()); + let (projected_field_ids, projected_columns) = self.resolve_projection(schema)?; + + let manifest_list_path = to_snapshot.manifest_list.as_ref().ok_or_else(|| { + IcebergError::Manifest( + "Snapshot has no manifest list (v1 format not supported)".to_string(), + ) + })?; + let manifest_list_data = self.storage.read(manifest_list_path).await?; + let manifest_entries = parse_manifest_list(&manifest_list_data)?; + + let mut added_tasks = Vec::new(); + let mut files_selected = 0; + let mut estimated_row_count = 0i64; + + for manifest_entry in &manifest_entries { + if manifest_entry.is_deletes() { + continue; // append-only: data manifests only + } + // A manifest written at or before `from` cannot hold files newer than + // `from` (every file it lists has data-seq <= the manifest's seq). + if manifest_entry.sequence_number <= from_seq { + continue; + } + + let manifest_data = self.storage.read(&manifest_entry.manifest_path).await?; + let data_file_entries = parse_manifest(&manifest_data)?; + + for entry in data_file_entries { + let eff_seq = effective_sequence_number( + entry.sequence_number, + manifest_entry.sequence_number, + ); + if eff_seq <= from_seq { + continue; // present at/before `from` + } + + let data_file = entry.data_file; + if let Some(filter) = &self.config.filter { + if !can_contain_file(filter, &data_file, schema) { + continue; + } + } + + files_selected += 1; + estimated_row_count += data_file.record_count; + added_tasks.push(FileScanTask::for_whole_file_with_schema( + data_file, + projected_field_ids.clone(), + self.config.filter.clone(), + Arc::clone(&schema_arc), + )); + } + } + + tracing::info!( + ?from_snapshot_id, + to_snapshot_id, + from_seq, + to_seq, + files_selected, + estimated_row_count, + "Incremental scan planning complete (append-only)" + ); + + Ok(IncrementalScanPlan { + added_tasks, + projected_columns, + projected_field_ids, + from_sequence_number: from_seq, + to_snapshot_id, + to_sequence_number: to_seq, + files_selected, + estimated_row_count, + }) + } + /// Resolve projection to field IDs and column names. fn resolve_projection(&self, schema: &Schema) -> Result<(Vec, Vec)> { match &self.config.projection { From 93702f41a3a164005023635c7f32a56444b3fae9 Mon Sep 17 00:00:00 2001 From: Christophe Di Prima Date: Fri, 26 Jun 2026 17:41:07 +0200 Subject: [PATCH 2/5] feat(api): r2rml incremental scan + snapshot helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds, on FlureeR2rmlProvider: - prepare_iceberg_scan(): shared setup (REST/Direct × GCS/S3 × creds × metadata cache). - read_scan_tasks(): bounded-parallel Parquet read of a task set. - current_snapshot_id(): the source table's current snapshot (materialize 'to' point). - scan_table_incremental(from, to): scans only data files ADDED in the snapshot window, via SendScanPlanner::plan_incremental. These back the materialization layer. NOTE: scan_table still inlines the same setup/read (TODO dedup once the incremental path is verified end-to-end). Co-Authored-By: Claude Opus 4.8 (1M context) --- fluree-db-api/src/graph_source/r2rml.rs | 282 ++++++++++++++++++++++++ 1 file changed, 282 insertions(+) diff --git a/fluree-db-api/src/graph_source/r2rml.rs b/fluree-db-api/src/graph_source/r2rml.rs index c18e70a037..c4da1794bf 100644 --- a/fluree-db-api/src/graph_source/r2rml.rs +++ b/fluree-db-api/src/graph_source/r2rml.rs @@ -459,6 +459,288 @@ impl<'a> FlureeR2rmlProvider<'a> { session: std::sync::Arc::new(super::catalog_session::IcebergCatalogSession::default()), } } + + /// Resolve a graph source's storage backend, parsed table metadata, and + /// metadata-location — the shared setup behind both full and incremental + /// scans (REST/Direct × GCS/S3 × credentials × caching). + /// + /// TODO(dedup): `scan_table` still inlines this same setup + the + /// `read_scan_tasks` read loop; once the incremental path is verified, + /// refactor `scan_table` to call these helpers and drop the duplication. + async fn prepare_iceberg_scan( + &self, + graph_source_id: &str, + table_name: &str, + ) -> QueryResult<(Arc, Arc, String)> { + // Look up the graph source record to get Iceberg connection info + let record = self + .fluree + .nameservice() + .lookup_graph_source(graph_source_id) + .await + .map_err(|e| QueryError::Internal(format!("Nameservice error: {e}")))? + .ok_or_else(|| { + QueryError::InvalidQuery(format!("Graph source '{graph_source_id}' not found")) + })?; + + let iceberg_config = IcebergGsConfig::from_json(&record.config).map_err(|e| { + QueryError::Internal(format!( + "Failed to parse Iceberg graph source config for '{graph_source_id}': {e}" + )) + })?; + iceberg_config.validate().map_err(|e| { + QueryError::InvalidQuery(format!( + "Invalid Iceberg graph source config for '{graph_source_id}': {e}" + )) + })?; + + use fluree_db_iceberg::catalog::parse_table_identifier; + use fluree_db_iceberg::config::CatalogConfig; + use fluree_db_iceberg::SendDirectCatalogClient; + + let table_id = if !table_name.is_empty() { + parse_table_identifier(table_name).map_err(|e| { + QueryError::Internal(format!( + "Failed to parse table identifier '{table_name}': {e}" + )) + })? + } else { + iceberg_config.table_identifier().map_err(|e| { + QueryError::Internal(format!("Failed to parse table identifier: {e}")) + })? + }; + + let (load_response, storage) = match &iceberg_config.catalog { + CatalogConfig::Rest { + uri, + warehouse, + auth, + .. + } => { + let auth_provider = auth.create_provider_arc().map_err(|e| { + QueryError::Internal(format!("Failed to create auth provider: {e}")) + })?; + let catalog_config = RestCatalogConfig { + uri: uri.clone(), + warehouse: warehouse.clone(), + ..Default::default() + }; + let catalog = + RestCatalogClient::new(catalog_config, auth_provider).map_err(|e| { + QueryError::Internal(format!("Failed to create catalog client: {e}")) + })?; + let load_response = catalog + .load_table(&table_id, iceberg_config.io.vended_credentials) + .await + .map_err(|e| { + QueryError::Internal(format!("Failed to load table from catalog: {e}")) + })?; + // GCS-backed tables read through this same S3 SDK path; the + // client is pinned to HTTP/1.1 (see `S3IcebergStorage`). Vended + // creds win, with the io config as fallback for region/endpoint/ + // path-style. + let storage = if let Some(ref credentials) = load_response.credentials { + S3IcebergStorage::from_vended_credentials( + credentials, + iceberg_config.io.s3_region.as_deref(), + iceberg_config.io.s3_endpoint.as_deref(), + iceberg_config.io.s3_path_style, + ) + .await + .map_err(|e| { + QueryError::Internal(format!("Failed to create S3 storage: {e}")) + })? + } else { + S3IcebergStorage::from_default_chain( + iceberg_config.io.s3_region.as_deref(), + iceberg_config.io.s3_endpoint.as_deref(), + iceberg_config.io.s3_path_style, + ) + .await + .map_err(|e| { + QueryError::Internal(format!("Failed to create S3 storage: {e}")) + })? + }; + (load_response, Arc::new(storage)) + } + CatalogConfig::Direct { table_location } => { + let storage: Arc = Arc::new( + S3IcebergStorage::from_default_chain( + iceberg_config.io.s3_region.as_deref(), + iceberg_config.io.s3_endpoint.as_deref(), + iceberg_config.io.s3_path_style, + ) + .await + .map_err(|e| { + QueryError::Internal(format!("Failed to create S3 storage: {e}")) + })?, + ); + let cache = self.fluree.r2rml_cache(); + let load_response = if let Some(metadata_location) = + cache.get_direct_metadata_location(table_location).await + { + fluree_db_iceberg::catalog::LoadTableResponse { + metadata_location, + config: std::collections::HashMap::default(), + credentials: None, + } + } else { + let direct_catalog = + SendDirectCatalogClient::new(table_location.clone(), Arc::clone(&storage)); + let load_response = + direct_catalog + .load_table(&table_id, false) + .await + .map_err(|e| { + QueryError::Internal(format!( + "Failed to resolve table metadata from {table_location}: {e}" + )) + })?; + cache + .put_direct_metadata_location( + table_location.clone(), + load_response.metadata_location.clone(), + ) + .await; + load_response + }; + (load_response, storage) + } + }; + + let cache = self.fluree.r2rml_cache(); + let metadata_location = load_response.metadata_location.clone(); + let metadata = if let Some(cached) = cache.get_metadata(&metadata_location).await { + cached + } else { + let metadata_bytes = storage + .as_ref() + .read(&metadata_location) + .await + .map_err(|e| QueryError::Internal(format!("Failed to read table metadata: {e}")))?; + let parsed = TableMetadata::from_json(&metadata_bytes).map_err(|e| { + QueryError::Internal(format!("Failed to parse table metadata: {e}")) + })?; + let metadata = Arc::new(parsed); + cache + .put_metadata(metadata_location.clone(), Arc::clone(&metadata)) + .await; + metadata + }; + + Ok((storage, metadata, metadata_location)) + } + + /// Read a set of scan tasks into column batches with bounded parallelism. + async fn read_scan_tasks( + &self, + storage: &Arc, + tasks: Vec, + ) -> QueryResult> { + if tasks.is_empty() { + return Ok(Vec::new()); + } + let footers = self.fluree.r2rml_cache().parquet_footers(); + let concurrency = iceberg_scan_concurrency(tasks.len()); + debug!( + files = tasks.len(), + concurrency, "reading Parquet files (bounded parallel)" + ); + use futures::stream::{StreamExt, TryStreamExt}; + let all_batches: Vec = futures::stream::iter(tasks) + .map(|task| { + let storage = Arc::clone(storage); + let footers = Arc::clone(&footers); + async move { + tokio::spawn(async move { + let reader = + SendParquetReader::with_cache(storage.as_ref(), footers.as_ref()); + reader.read_task(&task).await.map_err(|e| { + QueryError::Internal(format!( + "Failed to read Parquet file '{}': {e}", + task.data_file.file_path + )) + }) + }) + .await + .map_err(|e| QueryError::Internal(format!("Parquet read worker failed: {e}")))? + } + }) + .buffer_unordered(concurrency) + .try_collect::>>() + .await? + .into_iter() + .flatten() + .collect(); + Ok(all_batches) + } + + /// The source table's current snapshot id (the materialization "to" point), + /// or `None` if the table has no snapshots yet. + pub async fn current_snapshot_id( + &self, + graph_source_id: &str, + table_name: &str, + ) -> QueryResult> { + let (_storage, metadata, _loc) = self + .prepare_iceberg_scan(graph_source_id, table_name) + .await?; + Ok(metadata.current_snapshot().map(|s| s.snapshot_id)) + } + + /// Scan only the data files ADDED in the snapshot window + /// `(from_snapshot_id, to_snapshot_id]` (append-only incremental). + /// + /// `from_snapshot_id = None` reads the full live state of `to_snapshot_id` + /// (initial materialization). The caller must verify the window is + /// incremental-safe (`TableMetadata::window_is_append_only`, allowing + /// compaction) and fall back to a full scan otherwise. + pub async fn scan_table_incremental( + &self, + graph_source_id: &str, + table_name: &str, + projection: &[String], + from_snapshot_id: Option, + to_snapshot_id: i64, + ) -> QueryResult> { + let (storage, metadata, _loc) = self + .prepare_iceberg_scan(graph_source_id, table_name) + .await?; + + let schema = metadata + .current_schema() + .ok_or_else(|| QueryError::Internal("Table has no current schema".to_string()))?; + let projected_field_ids: Vec = if projection.is_empty() { + schema + .fields + .iter() + .filter(|f| !f.is_nested()) + .map(|f| f.id) + .collect() + } else { + projection + .iter() + .filter_map(|col| schema.field_by_name(col).map(|f| f.id)) + .collect() + }; + + let scan_config = ScanConfig::new().with_projection(projected_field_ids); + let planner = SendScanPlanner::new(storage.as_ref(), &metadata, scan_config); + let plan = planner + .plan_incremental(from_snapshot_id, to_snapshot_id) + .await + .map_err(|e| QueryError::Internal(format!("Failed to plan incremental scan: {e}")))?; + + info!( + from_snapshot_id = ?from_snapshot_id, + to_snapshot_id, + files = plan.files_selected, + estimated_rows = plan.estimated_row_count, + "Iceberg incremental scan plan created" + ); + + self.read_scan_tasks(&storage, plan.added_tasks).await + } } impl std::fmt::Debug for FlureeR2rmlProvider<'_> { From f8ce832cb295cbe49710e7ff5af8698adf48ab1a Mon Sep 17 00:00:00 2001 From: Christophe Di Prima Date: Tue, 30 Jun 2026 10:35:27 +0200 Subject: [PATCH 3/5] feat(iceberg): materialize R2RML graph sources into native ledgers + tracking worker Materialize an R2RML/Iceberg graph source into a native Fluree ledger so native features (BM25, vector/RAG, reasoning) can run over external tables. - Engine: Fluree::materialize_r2rml_graph_source(source, target, force_full) reuses the query-path R2RML term materializers (subject-IRI parity), aggregates one JSON-LD node per subject, and upserts into the target ledger. - Incremental: scan only files added in the (from, to] snapshot window when it is append/compaction-only (TableMetadata::window_is_incremental_safe); overwrite/delete or expired/branched history fall back to a full re-read. Compaction (replace) stays incremental because Iceberg preserves each row's data_sequence_number, so the sequence-number window excludes rewritten rows. - Watermark persisted in the target ledger (string-encoded snapshot id), written atomically in the same upsert; a no-delta poll commits nothing. - Tracking worker (MaterializeTrackingWorker): a Send polling task spawned on non-peer nodes that keeps tracked source->target jobs fresh on an interval. - Routes: POST /v1/fluree/iceberg/{materialize,track,untrack}, GET /v1/fluree/iceberg/tracking. Co-Authored-By: Claude Opus 4.8 (1M context) --- fluree-db-api/src/graph_source/mod.rs | 6 + fluree-db-api/src/graph_source/r2rml.rs | 93 +++++ .../src/graph_source/r2rml_materialize.rs | 385 ++++++++++++++++++ fluree-db-api/src/lib.rs | 12 +- fluree-db-api/src/materialize_worker.rs | 380 +++++++++++++++++ fluree-db-iceberg/src/metadata/table.rs | 70 ++++ fluree-db-server/src/routes/iceberg.rs | 278 +++++++++++++ fluree-db-server/src/routes/mod.rs | 12 +- fluree-db-server/src/state.rs | 34 ++ 9 files changed, 1266 insertions(+), 4 deletions(-) create mode 100644 fluree-db-api/src/graph_source/r2rml_materialize.rs create mode 100644 fluree-db-api/src/materialize_worker.rs diff --git a/fluree-db-api/src/graph_source/mod.rs b/fluree-db-api/src/graph_source/mod.rs index 66d9f9dbce..a84262f531 100644 --- a/fluree-db-api/src/graph_source/mod.rs +++ b/fluree-db-api/src/graph_source/mod.rs @@ -117,6 +117,9 @@ mod catalog_session; #[cfg(feature = "iceberg")] mod r2rml; +#[cfg(feature = "iceberg")] +mod r2rml_materialize; + // Re-export configuration types pub use config::Bm25CreateConfig; @@ -164,4 +167,7 @@ pub use provider::FlureeIndexProvider; #[cfg(feature = "iceberg")] pub use r2rml::FlureeR2rmlProvider; +#[cfg(feature = "iceberg")] +pub use r2rml_materialize::MaterializeResult; + // Helper functions are used internally by bm25.rs via direct module path diff --git a/fluree-db-api/src/graph_source/r2rml.rs b/fluree-db-api/src/graph_source/r2rml.rs index c4da1794bf..dfb06247fa 100644 --- a/fluree-db-api/src/graph_source/r2rml.rs +++ b/fluree-db-api/src/graph_source/r2rml.rs @@ -741,6 +741,99 @@ impl<'a> FlureeR2rmlProvider<'a> { self.read_scan_tasks(&storage, plan.added_tasks).await } + + /// Resolve the source table's current snapshot and read the rows to + /// materialize, choosing incremental vs full automatically. + /// + /// Returns `(to_snapshot_id, incremental, batches)`: + /// - `to_snapshot_id` — the source's current snapshot id (the new + /// watermark to persist), or `None` if the table has no snapshots yet + /// (nothing to materialize). + /// - `incremental` — whether an added-files-only scan was used (vs a full + /// read of the live table state). + /// + /// An incremental scan is used when `from_snapshot_id` is set **and** the + /// window `(from, to]` is incremental-safe — only `append`/`replace` + /// (compaction) operations, see + /// [`TableMetadata::window_is_incremental_safe`]. Otherwise a full scan of + /// `to` is performed: initial materialization (`from = None`), expired or + /// branched history, or a genuine `overwrite`/`delete` in the window (which + /// an added-files scan would miss). This keeps routine appends and periodic + /// compaction on the cheap incremental path while staying correct across + /// updates/deletes. + pub async fn scan_for_materialize( + &self, + graph_source_id: &str, + table_name: &str, + projection: &[String], + from_snapshot_id: Option, + ) -> QueryResult<(Option, bool, Vec)> { + let (storage, metadata, _loc) = self + .prepare_iceberg_scan(graph_source_id, table_name) + .await?; + + let Some(to_snapshot_id) = metadata.current_snapshot().map(|s| s.snapshot_id) else { + // Table has no snapshots: nothing to materialize. + return Ok((None, false, Vec::new())); + }; + + let schema = metadata + .current_schema() + .ok_or_else(|| QueryError::Internal("Table has no current schema".to_string()))?; + let projected_field_ids: Vec = if projection.is_empty() { + schema + .fields + .iter() + .filter(|f| !f.is_nested()) + .map(|f| f.id) + .collect() + } else { + projection + .iter() + .filter_map(|col| schema.field_by_name(col).map(|f| f.id)) + .collect() + }; + + let incremental = from_snapshot_id.is_some() + && metadata + .window_is_incremental_safe(from_snapshot_id, to_snapshot_id) + .unwrap_or(false); + + let scan_config = ScanConfig::new().with_projection(projected_field_ids); + let planner = SendScanPlanner::new(storage.as_ref(), &metadata, scan_config); + + let tasks = if incremental { + let plan = planner + .plan_incremental(from_snapshot_id, to_snapshot_id) + .await + .map_err(|e| { + QueryError::Internal(format!("Failed to plan incremental scan: {e}")) + })?; + info!( + from_snapshot_id = ?from_snapshot_id, + to_snapshot_id, + files = plan.files_selected, + estimated_rows = plan.estimated_row_count, + "materialize: incremental (added-files) scan plan" + ); + plan.added_tasks + } else { + let plan = planner + .plan_scan() + .await + .map_err(|e| QueryError::Internal(format!("Failed to plan full scan: {e}")))?; + info!( + to_snapshot_id, + files = plan.files_selected, + estimated_rows = plan.estimated_row_count, + "materialize: full scan plan" + ); + plan.tasks + }; + + let batches = self.read_scan_tasks(&storage, tasks).await?; + Ok((Some(to_snapshot_id), incremental, batches)) + } } impl std::fmt::Debug for FlureeR2rmlProvider<'_> { diff --git a/fluree-db-api/src/graph_source/r2rml_materialize.rs b/fluree-db-api/src/graph_source/r2rml_materialize.rs new file mode 100644 index 0000000000..9275fe3e80 --- /dev/null +++ b/fluree-db-api/src/graph_source/r2rml_materialize.rs @@ -0,0 +1,385 @@ +//! Materialize an R2RML / Iceberg graph source into a native Fluree ledger. +//! +//! The query path reads a graph source on the fly via `CONSTRUCT`-style +//! template expansion. Native Fluree features (BM25 full-text search, vector / +//! RAG, reasoning) operate only on facts committed to a *native* ledger, so to +//! make those work over an external Iceberg table we **materialize** it: read +//! the source rows, expand the R2RML subject / predicate / object maps into RDF +//! terms exactly as the query path does, group them per subject, and `upsert` +//! them into a target ledger. +//! +//! Refreshes are incremental when safe. The caller passes the last materialized +//! snapshot id as `from_snapshot_id`; if the source's `(from, to]` window is +//! append/compaction-only (see +//! [`window_is_incremental_safe`](fluree_db_iceberg::metadata::TableMetadata::window_is_incremental_safe)), +//! only the *added* rows are read and upserted. The returned `to_snapshot_id` is +//! the new watermark for the caller to persist — watermark storage is left to +//! the caller (the tracking worker) so this stays free of storage assumptions. +//! +//! This module is only available with the `iceberg` feature. + +use std::collections::BTreeMap; + +use serde_json::{json, Map, Value as JsonValue}; + +use fluree_db_iceberg::io::ColumnBatch; +use fluree_db_query::r2rml::R2rmlProvider; +use fluree_db_r2rml::mapping::TriplesMap; +use fluree_db_r2rml::materialize::{ + materialize_object_from_batch, materialize_subject_from_batch, RdfTerm, +}; +use fluree_vocab::UnresolvedDatatypeConstraint; + +use crate::graph_source::FlureeR2rmlProvider; +use crate::{ApiError, Fluree, Result}; + +/// Subject-IRI prefix for the per-source materialization watermark stored in the +/// target ledger. The full subject is `{PREFIX}{source_graph_source_id}` so one +/// target ledger can track several sources independently. +const WATERMARK_SUBJECT_PREFIX: &str = "urn:fluree:materialize-state:"; +/// Predicate holding the last materialized source snapshot id (stored as a +/// string to preserve full i64 precision for 19-digit snapshot ids). +const WATERMARK_SNAPSHOT_PRED: &str = "urn:fluree:materialize#lastSnapshotId"; +/// Predicate recording which source the watermark belongs to (informational). +const WATERMARK_SOURCE_PRED: &str = "urn:fluree:materialize#source"; + +/// Outcome of one materialization pass. +#[derive(Debug, Clone)] +pub struct MaterializeResult { + /// The source snapshot that was materialized — persisted as the watermark in + /// the target ledger for the next incremental refresh. `None` when the source + /// table has no snapshots yet (nothing was materialized). + pub to_snapshot_id: Option, + /// The watermark this pass started from (the previously-materialized source + /// snapshot id), or `None` for an initial / forced-full materialization. + pub from_snapshot_id: Option, + /// Whether an incremental (added-files-only) scan was used. `false` means a + /// full re-read (initial run, expired/branched history, or a window + /// containing a genuine `overwrite`/`delete`). + pub incremental: bool, + /// Whether anything was committed (data and/or an advanced watermark). A + /// no-delta poll (`from == to`) returns `false` — no commit churn. + pub committed: bool, + /// Number of source rows read across all batches. + pub rows_read: usize, + /// Number of distinct subject nodes upserted. + pub subjects_upserted: usize, +} + +impl Fluree { + /// Materialize an R2RML / Iceberg graph source into a native ledger. + /// + /// Resolves the starting point from the **watermark persisted in the target + /// ledger** (the last materialized source snapshot id) — unless `force_full` + /// is set, which ignores it and re-reads the whole live table. Reads the + /// source (incrementally when the `(from, to]` window is append/compaction + /// only, else full), expands each row through the source's compiled R2RML + /// mapping into RDF terms, and `upsert`s one JSON-LD node per subject into + /// `target_ledger_id` (created if absent). The new watermark is written in + /// the **same** `upsert` commit, so it advances atomically with the data and + /// the operation is idempotent. Callers track nothing; a bare re-run resumes + /// incrementally. This makes the call safe to invoke on a timer (the + /// tracking worker does exactly that). + /// + /// A no-delta poll (`from == to`) commits nothing and returns + /// `committed = false`. + /// + /// **Deletion safety.** An added-files scan captures inserts (and in-place + /// updates land via `upsert`), but it cannot see row *deletions*. The window + /// check therefore forces a full re-read whenever an `overwrite`/`delete` + /// snapshot appears, so deletes are never silently missed — while routine + /// appends and periodic compaction stay on the cheap incremental path. A + /// full re-read still only `upsert`s present rows; reconciling subjects that + /// vanished entirely (set-difference retraction) is a separate follow-up. + pub async fn materialize_r2rml_graph_source( + &self, + source_graph_source_id: &str, + target_ledger_id: &str, + force_full: bool, + ) -> Result { + let provider = FlureeR2rmlProvider::new(self); + + // 1. Compiled R2RML mapping (subject / predicate / object maps). + let mapping = provider + .compiled_mapping(source_graph_source_id, None) + .await?; + if mapping.triples_maps.is_empty() { + return Err(ApiError::Config(format!( + "Graph source '{source_graph_source_id}' has no R2RML triples maps" + ))); + } + + // 2. Resolve the starting watermark from the target ledger (unless the + // caller forces a full re-read). + let from_snapshot_id = if force_full { + None + } else { + self.materialize_watermark(target_ledger_id, source_graph_source_id) + .await? + }; + + // 3. Read source rows (incremental when safe) per logical table and + // aggregate all triples per subject IRI into one JSON-LD node. The + // BTreeMap keeps subjects in a stable order (deterministic txn). + let mut subjects: BTreeMap = BTreeMap::new(); + let mut rows_read = 0usize; + let mut to_snapshot_id: Option = None; + // The pass is incremental only if *every* table's scan was incremental + // (and we resolved a watermark to begin with). + let mut incremental_all = from_snapshot_id.is_some(); + let mut any_table = false; + + for tm in mapping.triples_maps.values() { + let Some(table_name) = tm.table_name() else { + // Subject-only / constant-IRI maps with no logical table. + continue; + }; + any_table = true; + + let (to_id, incremental, batches) = provider + .scan_for_materialize(source_graph_source_id, table_name, &[], from_snapshot_id) + .await?; + to_snapshot_id = to_id; + incremental_all = incremental_all && incremental; + + for batch in &batches { + for row in 0..batch.num_rows { + rows_read += 1; + materialize_row_into(tm, batch, row, &mut subjects)?; + } + } + } + + let incremental = any_table && incremental_all; + let subjects_upserted = subjects.len(); + + // 4. Decide whether anything needs committing. Nothing changed when the + // source has no snapshot, or the watermark already equals `to` and no + // rows were produced (a no-delta poll) — skip the commit entirely. + let watermark_unchanged = to_snapshot_id.is_none() || to_snapshot_id == from_snapshot_id; + if subjects.is_empty() && watermark_unchanged { + return Ok(MaterializeResult { + to_snapshot_id, + from_snapshot_id, + incremental, + committed: false, + rows_read, + subjects_upserted: 0, + }); + } + + // 5. Build the JSON-LD insert array: one node per subject, plus the + // advanced watermark node (committed atomically with the data). + let mut nodes: Vec = + subjects.into_values().map(SubjectNode::into_json).collect(); + if let Some(to) = to_snapshot_id { + nodes.push(watermark_node(source_graph_source_id, to)); + } + let doc = JsonValue::Array(nodes); + + // 6. Load-or-create the target ledger and upsert. + let ledger = if self.ledger_exists(target_ledger_id).await? { + self.ledger(target_ledger_id).await? + } else { + self.create_ledger(target_ledger_id).await? + }; + self.upsert(ledger, &doc).await?; + + Ok(MaterializeResult { + to_snapshot_id, + from_snapshot_id, + incremental, + committed: true, + rows_read, + subjects_upserted, + }) + } + + /// Read the materialization watermark (last materialized source snapshot id) + /// for `source_graph_source_id` from the target ledger. Returns `None` if the + /// target ledger does not exist yet or carries no watermark for that source. + pub async fn materialize_watermark( + &self, + target_ledger_id: &str, + source_graph_source_id: &str, + ) -> Result> { + if !self.ledger_exists(target_ledger_id).await? { + return Ok(None); + } + let db = self.db(target_ledger_id).await?; + + let subject = format!("{WATERMARK_SUBJECT_PREFIX}{source_graph_source_id}"); + let mut where_obj = Map::new(); + where_obj.insert("@id".to_string(), JsonValue::String(subject)); + where_obj.insert( + WATERMARK_SNAPSHOT_PRED.to_string(), + JsonValue::String("?v".to_string()), + ); + let query = json!({ "select": ["?v"], "where": JsonValue::Object(where_obj) }); + + let result = self.query(&db, &query).await?; + let json = result + .to_jsonld(&db.snapshot) + .map_err(|e| ApiError::Internal(format!("Failed to format watermark query: {e}")))?; + Ok(extract_first_i64(&json)) + } +} + +/// Build the watermark JSON-LD node (`@id` + last snapshot id + source). Stored +/// as a string to preserve full i64 precision; `upsert` retracts-then-asserts so +/// the watermark advances cleanly in place. +fn watermark_node(source_graph_source_id: &str, to_snapshot_id: i64) -> JsonValue { + let mut node = Map::new(); + node.insert( + "@id".to_string(), + JsonValue::String(format!( + "{WATERMARK_SUBJECT_PREFIX}{source_graph_source_id}" + )), + ); + node.insert( + WATERMARK_SNAPSHOT_PRED.to_string(), + JsonValue::String(to_snapshot_id.to_string()), + ); + node.insert( + WATERMARK_SOURCE_PRED.to_string(), + JsonValue::String(source_graph_source_id.to_string()), + ); + JsonValue::Object(node) +} + +/// Find the first value anywhere in a JSON-LD query result that parses as an +/// i64 (the watermark query selects exactly one string value, so this is +/// unambiguous). Tolerant of the formatter's row nesting. +fn extract_first_i64(v: &JsonValue) -> Option { + match v { + JsonValue::Array(a) => a.iter().find_map(extract_first_i64), + JsonValue::Object(o) => o.values().find_map(extract_first_i64), + JsonValue::String(s) => s.parse::().ok(), + JsonValue::Number(n) => n.as_i64(), + _ => None, + } +} + +/// Accumulates all triples for a single subject IRI before emitting one JSON-LD +/// node. Predicates collect multiple object values (combined into an array on +/// output) and `@type` classes are de-duplicated. +struct SubjectNode { + id: String, + types: Vec, + /// predicate IRI -> object values (`@id` refs, value objects, or scalars). + predicates: BTreeMap>, +} + +impl SubjectNode { + fn new(id: String) -> Self { + Self { + id, + types: Vec::new(), + predicates: BTreeMap::new(), + } + } + + fn add_type(&mut self, class: &str) { + if !self.types.iter().any(|c| c == class) { + self.types.push(class.to_string()); + } + } + + fn add_object(&mut self, predicate: &str, value: JsonValue) { + let entry = self.predicates.entry(predicate.to_string()).or_default(); + if !entry.iter().any(|v| v == &value) { + entry.push(value); + } + } + + fn into_json(self) -> JsonValue { + let mut node = Map::new(); + node.insert("@id".to_string(), JsonValue::String(self.id)); + if !self.types.is_empty() { + node.insert( + "@type".to_string(), + JsonValue::Array(self.types.into_iter().map(JsonValue::String).collect()), + ); + } + for (pred, mut vals) in self.predicates { + let v = if vals.len() == 1 { + vals.pop().expect("len == 1") + } else { + JsonValue::Array(vals) + }; + node.insert(pred, v); + } + JsonValue::Object(node) + } +} + +/// Expand one source row through a triples map and merge its triples into the +/// per-subject accumulator. Reuses the exact `term.rs` materializers so the +/// subject IRIs match the query path byte-for-byte. +fn materialize_row_into( + tm: &TriplesMap, + batch: &ColumnBatch, + row: usize, + subjects: &mut BTreeMap, +) -> Result<()> { + let subject_term = materialize_subject_from_batch(&tm.subject_map, batch, row) + .map_err(|e| ApiError::Internal(format!("R2RML subject materialization failed: {e}")))?; + let subject_iri = match subject_term { + Some(RdfTerm::Iri(iri)) => iri, + // Null subject column (skip), blank-node subjects (no stable identity to + // upsert), or a literal (term.rs already rejects this) -> skip the row. + Some(RdfTerm::BlankNode(_) | RdfTerm::Literal { .. }) | None => return Ok(()), + }; + + let node = subjects + .entry(subject_iri.clone()) + .or_insert_with(|| SubjectNode::new(subject_iri)); + + // rr:class -> @type + for class in &tm.subject_map.classes { + node.add_type(class); + } + + // predicate-object maps (constant predicates only; RefObjectMap joins are + // resolved at query time, not yet during materialization). + for pom in &tm.predicate_object_maps { + let Some(predicate) = pom.predicate_map.as_constant() else { + continue; + }; + let obj = materialize_object_from_batch(&pom.object_map, batch, row) + .map_err(|e| ApiError::Internal(format!("R2RML object materialization failed: {e}")))?; + if let Some(term) = obj { + node.add_object(predicate, rdf_term_to_jsonld(term)); + } + } + + Ok(()) +} + +/// Convert an [`RdfTerm`] into its JSON-LD object representation. +fn rdf_term_to_jsonld(term: RdfTerm) -> JsonValue { + match term { + RdfTerm::Iri(iri) | RdfTerm::BlankNode(iri) => { + let mut m = Map::new(); + m.insert("@id".to_string(), JsonValue::String(iri)); + JsonValue::Object(m) + } + RdfTerm::Literal { value, dtc } => match dtc { + // Plain literal (xsd:string) -> bare JSON string, matching the + // query path's default datatype. + None => JsonValue::String(value), + Some(UnresolvedDatatypeConstraint::LangTag(lang)) => { + let mut m = Map::new(); + m.insert("@value".to_string(), JsonValue::String(value)); + m.insert("@language".to_string(), JsonValue::String(lang.to_string())); + JsonValue::Object(m) + } + Some(UnresolvedDatatypeConstraint::Explicit(dt)) => { + let mut m = Map::new(); + m.insert("@value".to_string(), JsonValue::String(value)); + m.insert("@type".to_string(), JsonValue::String(dt.to_string())); + JsonValue::Object(m) + } + }, + } +} diff --git a/fluree-db-api/src/lib.rs b/fluree-db-api/src/lib.rs index 6510b4a506..7a48cca03c 100644 --- a/fluree-db-api/src/lib.rs +++ b/fluree-db-api/src/lib.rs @@ -64,6 +64,8 @@ mod inline_ontology; mod inline_shapes; mod ledger; pub mod ledger_info; +#[cfg(feature = "iceberg")] +pub mod materialize_worker; mod merge; mod merge_preview; pub mod nameservice_query; @@ -189,14 +191,20 @@ pub use view::{ #[cfg(feature = "iceberg")] pub use graph_source::{ - CatalogMode, FlureeR2rmlProvider, IcebergCreateConfig, IcebergCreateResult, R2rmlCreateConfig, - R2rmlCreateResult, R2rmlMappingInput, RestCatalogMode, + CatalogMode, FlureeR2rmlProvider, IcebergCreateConfig, IcebergCreateResult, MaterializeResult, + R2rmlCreateConfig, R2rmlCreateResult, R2rmlMappingInput, RestCatalogMode, }; pub use bm25_worker::{ Bm25MaintenanceWorker, Bm25WorkerConfig, Bm25WorkerHandle, Bm25WorkerState, Bm25WorkerStats, }; +#[cfg(feature = "iceberg")] +pub use materialize_worker::{ + MaterializeTrackingWorker, MaterializeWorkerConfig, MaterializeWorkerHandle, + MaterializeWorkerStats, +}; + #[cfg(feature = "vector")] pub use vector_worker::{ VectorMaintenanceWorker, VectorWorkerConfig, VectorWorkerHandle, VectorWorkerState, diff --git a/fluree-db-api/src/materialize_worker.rs b/fluree-db-api/src/materialize_worker.rs new file mode 100644 index 0000000000..17970a135e --- /dev/null +++ b/fluree-db-api/src/materialize_worker.rs @@ -0,0 +1,380 @@ +//! Background tracking worker for R2RML / Iceberg materialization. +//! +//! Watches a set of `(source graph source → target ledger)` jobs and keeps each +//! target ledger fresh by periodically calling +//! [`Fluree::materialize_r2rml_graph_source`](crate::Fluree::materialize_r2rml_graph_source). +//! That call resolves the watermark persisted in the target ledger and refreshes +//! incrementally (append/compaction-only windows) or fully (first run or a +//! non-incremental-safe window), committing nothing on a no-delta poll. The +//! worker therefore needs no state of its own beyond the set of jobs — the +//! watermark lives in the ledger, so a no-delta poll is a single metadata read. +//! +//! Each job carries its **own poll interval** (set per `track` call; defaults to +//! the worker's configured interval). The loop wakes on a fine base granularity +//! and polls only the jobs whose `next_due` has arrived, so jobs on different +//! timers are independent. +//! +//! Unlike [`crate::bm25_worker`] (event-driven, single-threaded `Rc`/`RefCell`), +//! this worker is **polling** and `Send`: it owns an `Arc` and shares its +//! job set / stop flag / stats behind `Arc>`, so it spawns with a plain +//! `tokio::spawn`. Iceberg emits no commit events, so polling is the only option. +//! +//! Only available with the `iceberg` feature. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use tokio::time::{self, Duration, MissedTickBehavior}; +use tracing::{debug, info, warn}; + +use crate::Fluree; + +/// Configuration for the materialization tracking worker. +#[derive(Debug, Clone)] +pub struct MaterializeWorkerConfig { + /// Default poll interval for a tracked job when `track` does not specify one. + pub poll_interval: Duration, +} + +impl Default for MaterializeWorkerConfig { + fn default() -> Self { + Self { + poll_interval: Duration::from_secs(30), + } + } +} + +/// Statistics for the tracking worker. +#[derive(Debug, Clone, Default)] +pub struct MaterializeWorkerStats { + /// Number of per-job poll attempts. + pub polls: u64, + /// Polls that committed data and/or an advanced watermark. + pub syncs_committed: u64, + /// Polls that found no delta (nothing committed). + pub syncs_noop: u64, + /// Polls that errored. + pub syncs_failed: u64, + /// Currently tracked jobs. + pub tracked_jobs: usize, +} + +/// A tracked materialization job: read `source`, materialize into `target`. +#[derive(Debug, Clone)] +struct TrackedJob { + source: String, + target: String, + /// How often this job is polled. + interval: Duration, + /// When this job is next due for a poll. + next_due: Instant, +} + +/// Public view of a tracked job for status endpoints. +#[derive(Debug, Clone)] +pub struct JobInfo { + pub source: String, + pub target: String, + /// This job's poll interval, in seconds. + pub poll_interval_secs: u64, +} + +type JobKey = (String, String); + +/// Handle to a running [`MaterializeTrackingWorker`]: register / unregister +/// jobs, read stats, and request stop. Cheap to clone (all shared state is +/// behind `Arc`). Shared between the spawned worker task and request handlers. +#[derive(Clone)] +pub struct MaterializeWorkerHandle { + tracked: Arc>>, + /// Poll interval applied when `track` is called without an explicit one. + default_interval: Duration, + stop: Arc, + stats: Arc>, +} + +impl MaterializeWorkerHandle { + /// Start tracking `source → target` (idempotent). `interval` is this job's + /// own poll cadence; `None` uses the worker's configured default. Returns the + /// effective interval. The next poll fires one interval from now (the caller + /// typically runs an immediate first sync itself). + pub fn track(&self, source: &str, target: &str, interval: Option) -> Duration { + let interval = interval.unwrap_or(self.default_interval); + let key = (source.to_string(), target.to_string()); + let mut tracked = self.tracked.lock().expect("materialize worker mutex"); + tracked.insert( + key, + TrackedJob { + source: source.to_string(), + target: target.to_string(), + interval, + next_due: Instant::now() + interval, + }, + ); + let n = tracked.len(); + drop(tracked); + self.stats.lock().expect("stats mutex").tracked_jobs = n; + info!( + source, + target, + tracked_jobs = n, + poll_interval_secs = interval.as_secs(), + "materialize tracking added" + ); + interval + } + + /// Stop tracking `source → target`. Returns whether a job was removed. The + /// already-materialized data and watermark in the target ledger are left + /// untouched. + pub fn untrack(&self, source: &str, target: &str) -> bool { + let key = (source.to_string(), target.to_string()); + let mut tracked = self.tracked.lock().expect("materialize worker mutex"); + let removed = tracked.remove(&key).is_some(); + let n = tracked.len(); + drop(tracked); + self.stats.lock().expect("stats mutex").tracked_jobs = n; + if removed { + info!( + source, + target, + tracked_jobs = n, + "materialize tracking removed" + ); + } + removed + } + + /// The currently tracked `(source, target)` pairs. + pub fn tracked_jobs(&self) -> Vec { + self.tracked + .lock() + .expect("materialize worker mutex") + .keys() + .cloned() + .collect() + } + + /// The currently tracked jobs with their per-job poll intervals. + pub fn job_infos(&self) -> Vec { + self.tracked + .lock() + .expect("materialize worker mutex") + .values() + .map(|j| JobInfo { + source: j.source.clone(), + target: j.target.clone(), + poll_interval_secs: j.interval.as_secs(), + }) + .collect() + } + + /// Snapshot of worker statistics. + pub fn stats(&self) -> MaterializeWorkerStats { + self.stats.lock().expect("stats mutex").clone() + } + + /// Request the worker to stop after the current cycle. + pub fn stop(&self) { + self.stop.store(true, Ordering::SeqCst); + info!("materialize tracking worker stop requested"); + } + + /// Whether stop has been requested. + pub fn is_stopped(&self) -> bool { + self.stop.load(Ordering::SeqCst) + } + + /// Jobs whose `next_due` has arrived; reschedules each to `now + interval` + /// before returning it (so a slow poll can't double-schedule the same job). + fn take_due_jobs(&self) -> Vec { + let now = Instant::now(); + let mut tracked = self.tracked.lock().expect("materialize worker mutex"); + let mut due = Vec::new(); + for job in tracked.values_mut() { + if job.next_due <= now { + job.next_due = now + job.interval; + due.push(job.clone()); + } + } + due + } +} + +/// Polling worker that keeps tracked target ledgers fresh. +pub struct MaterializeTrackingWorker { + fluree: Arc, + config: MaterializeWorkerConfig, + handle: MaterializeWorkerHandle, +} + +impl MaterializeTrackingWorker { + /// Create a worker with default config. + pub fn new(fluree: Arc) -> Self { + Self::with_config(fluree, MaterializeWorkerConfig::default()) + } + + /// Create a worker with a custom config. + pub fn with_config(fluree: Arc, config: MaterializeWorkerConfig) -> Self { + Self { + fluree, + handle: MaterializeWorkerHandle { + tracked: Arc::new(Mutex::new(BTreeMap::new())), + default_interval: config.poll_interval, + stop: Arc::new(AtomicBool::new(false)), + stats: Arc::new(Mutex::new(MaterializeWorkerStats::default())), + }, + config, + } + } + + /// Get a clonable handle to register jobs / read stats / stop the worker. + pub fn handle(&self) -> MaterializeWorkerHandle { + self.handle.clone() + } + + /// Run the poll loop until [`MaterializeWorkerHandle::stop`] is requested. + /// Spawn this with `tokio::spawn(worker.run())`. + pub async fn run(self) { + // Wake on a fine base granularity and poll whichever jobs are due; each + // job's own interval decides how often it actually runs. Cap the tick at + // 1s so a newly-added short-interval job (and a stop request) are noticed + // promptly, but never faster than the default interval (small tests). + let tick = self.config.poll_interval.min(Duration::from_secs(1)); + let mut ticker = time::interval(tick); + // A slow materialize shouldn't cause a burst of catch-up ticks. + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + info!( + default_poll_interval_secs = self.config.poll_interval.as_secs(), + tick_granularity_ms = u64::try_from(tick.as_millis()).unwrap_or(u64::MAX), + "materialize tracking worker started" + ); + + loop { + ticker.tick().await; + if self.handle.is_stopped() { + info!("materialize tracking worker stopped"); + break; + } + + for job in self.handle.take_due_jobs() { + if self.handle.is_stopped() { + break; + } + self.poll_job(&job).await; + } + } + } + + /// Poll one job: a single idempotent incremental materialize. Commits only + /// when the source advanced; logs and counts the outcome. + async fn poll_job(&self, job: &TrackedJob) { + self.bump(|s| s.polls += 1); + match self + .fluree + .materialize_r2rml_graph_source(&job.source, &job.target, false) + .await + { + Ok(result) if result.committed => { + self.bump(|s| s.syncs_committed += 1); + info!( + source = %job.source, + target = %job.target, + from_snapshot_id = ?result.from_snapshot_id, + to_snapshot_id = ?result.to_snapshot_id, + incremental = result.incremental, + rows_read = result.rows_read, + subjects_upserted = result.subjects_upserted, + "materialize tracking synced" + ); + } + Ok(_) => { + self.bump(|s| s.syncs_noop += 1); + debug!(source = %job.source, target = %job.target, "materialize tracking: no delta"); + } + Err(e) => { + self.bump(|s| s.syncs_failed += 1); + warn!(source = %job.source, target = %job.target, error = %e, "materialize tracking sync failed"); + } + } + } + + fn bump(&self, f: impl FnOnce(&mut MaterializeWorkerStats)) { + let mut stats = self.handle.stats.lock().expect("stats mutex"); + f(&mut stats); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + impl MaterializeWorkerHandle { + /// Build a bare handle (no worker / `Fluree`) for scheduling tests. + fn for_test(default_interval: Duration) -> Self { + Self { + tracked: Arc::new(Mutex::new(BTreeMap::new())), + default_interval, + stop: Arc::new(AtomicBool::new(false)), + stats: Arc::new(Mutex::new(MaterializeWorkerStats::default())), + } + } + } + + #[test] + fn track_uses_explicit_interval_else_default() { + let h = MaterializeWorkerHandle::for_test(Duration::from_secs(30)); + assert_eq!( + h.track("s:main", "t:main", Some(Duration::from_secs(300))), + Duration::from_secs(300), + "explicit interval is used and returned" + ); + assert_eq!( + h.track("s2:main", "t:main", None), + Duration::from_secs(30), + "None falls back to the worker default" + ); + let infos = h.job_infos(); + assert_eq!( + infos + .iter() + .find(|j| j.source == "s:main") + .unwrap() + .poll_interval_secs, + 300 + ); + assert_eq!( + infos + .iter() + .find(|j| j.source == "s2:main") + .unwrap() + .poll_interval_secs, + 30 + ); + } + + #[test] + fn take_due_jobs_respects_next_due_and_reschedules() { + let h = MaterializeWorkerHandle::for_test(Duration::from_secs(1)); + h.track("s:main", "t:main", Some(Duration::from_secs(3600))); + assert!( + h.take_due_jobs().is_empty(), + "a just-tracked job is not due until now + interval" + ); + // Rewind next_due into the past to make it due. + { + let mut tracked = h.tracked.lock().unwrap(); + tracked + .get_mut(&("s:main".to_string(), "t:main".to_string())) + .unwrap() + .next_due = Instant::now() - Duration::from_secs(1); + } + assert_eq!(h.take_due_jobs().len(), 1, "overdue job is returned"); + assert!( + h.take_due_jobs().is_empty(), + "taking a due job reschedules it ~1h out" + ); + } +} diff --git a/fluree-db-iceberg/src/metadata/table.rs b/fluree-db-iceberg/src/metadata/table.rs index 04e35744d5..20e44b8f6c 100644 --- a/fluree-db-iceberg/src/metadata/table.rs +++ b/fluree-db-iceberg/src/metadata/table.rs @@ -166,6 +166,34 @@ impl TableMetadata { let window = self.snapshot_window(from_id, to_id)?; Ok(window.iter().all(|s| s.operation() == Some("append"))) } + + /// Whether every snapshot in `(from_id, to_id]` is incremental-safe for an + /// **added-files-only** scan — i.e. each is an `append` (new data files + /// only) or a `replace` (compaction: files rewritten without any logical + /// change). A `replace` is safe because Iceberg preserves each row's + /// `data_sequence_number` through compaction, so the sequence-number window + /// `(from.seq, to.seq]` still excludes the rewritten old rows — compaction + /// never surfaces as a spurious "added" row. `overwrite`/`delete` + /// operations carry row-level updates and deletions an added-files scan + /// cannot see, so they are NOT incremental-safe (the caller must + /// full-refresh). A snapshot with no recorded operation is treated as + /// unsafe (fail safe). Propagates `snapshot_window` errors + /// (unknown/expired/non-ancestor). + /// + /// This is the check the materialization path uses: it keeps routine + /// appends *and* periodic compaction on the cheap incremental path, while + /// still falling back to a full re-read whenever genuine updates/deletes + /// (overwrite/delete) appear in the window. + pub fn window_is_incremental_safe( + &self, + from_id: Option, + to_id: i64, + ) -> crate::error::Result { + let window = self.snapshot_window(from_id, to_id)?; + Ok(window + .iter() + .all(|s| matches!(s.operation(), Some("append" | "replace")))) + } } /// Schema definition. @@ -492,4 +520,46 @@ mod tests { ]); assert!(!no_op.window_is_append_only(Some(1), 2).unwrap()); } + + #[test] + fn window_is_incremental_safe_allows_compaction_but_not_overwrite() { + // append + compaction (replace) is incremental-safe: compaction + // preserves data_sequence_number, so the seq-number window still + // excludes the rewritten old rows. + let append_then_compact = meta_with(vec![ + snap(1, None, 1, Some("append")), + snap(2, Some(1), 2, Some("append")), + snap(3, Some(2), 3, Some("replace")), + ]); + assert!(append_then_compact + .window_is_incremental_safe(Some(1), 3) + .unwrap()); + // ...but a pure replace window must NOT be treated as append-only. + assert!(!append_then_compact + .window_is_append_only(Some(1), 3) + .unwrap()); + + // overwrite carries row-level updates/deletes -> not incremental-safe. + let with_overwrite = meta_with(vec![ + snap(1, None, 1, Some("append")), + snap(2, Some(1), 2, Some("overwrite")), + ]); + assert!(!with_overwrite + .window_is_incremental_safe(Some(1), 2) + .unwrap()); + + // delete carries row removals -> not incremental-safe. + let with_delete = meta_with(vec![ + snap(1, None, 1, Some("append")), + snap(2, Some(1), 2, Some("delete")), + ]); + assert!(!with_delete.window_is_incremental_safe(Some(1), 2).unwrap()); + + // unrecorded operation -> fail safe. + let no_op = meta_with(vec![ + snap(1, None, 1, Some("append")), + snap(2, Some(1), 2, None), + ]); + assert!(!no_op.window_is_incremental_safe(Some(1), 2).unwrap()); + } } diff --git a/fluree-db-server/src/routes/iceberg.rs b/fluree-db-server/src/routes/iceberg.rs index 36ce55fa54..e7176c47fe 100644 --- a/fluree-db-server/src/routes/iceberg.rs +++ b/fluree-db-server/src/routes/iceberg.rs @@ -180,6 +180,284 @@ async fn iceberg_map_local(state: Arc, request: Request) -> Result, + /// The source snapshot now materialized (the persisted watermark). + pub to_snapshot_id: Option, + /// Whether an incremental (added-files-only) scan was used. + pub incremental: bool, + /// Whether anything was committed (false on a no-delta poll). + pub committed: bool, + pub rows_read: usize, + pub subjects_upserted: usize, +} + +/// Materialize an R2RML / Iceberg graph source into a native ledger. +/// +/// POST /v1/fluree/iceberg/materialize +pub async fn iceberg_materialize(State(state): State>, request: Request) -> Response { + if state.config.server_role == ServerRole::Peer { + return forward_write_request(&state, request).await; + } + + iceberg_materialize_local(state, request) + .await + .into_response() +} + +async fn iceberg_materialize_local( + state: Arc, + request: Request, +) -> Result { + let headers = FlureeHeaders::from_headers(request.headers())?; + + let body_bytes = axum::body::to_bytes(request.into_body(), 50 * 1024 * 1024) + .await + .map_err(|e| ServerError::bad_request(format!("Failed to read body: {e}")))?; + let req: IcebergMaterializeRequest = serde_json::from_slice(&body_bytes) + .map_err(|e| ServerError::bad_request(format!("Invalid JSON: {e}")))?; + + let request_id = extract_request_id(&headers.raw, &state.telemetry_config); + let trace_id = extract_trace_id(&headers.raw); + + let span = create_request_span( + "iceberg:materialize", + request_id.as_deref(), + trace_id.as_deref(), + Some(&req.source), + None, + None, + ); + async move { + tracing::info!( + status = "start", + source = %req.source, + target = %req.target, + force_full = req.force_full, + "iceberg materialize requested" + ); + + let result = state + .fluree + .materialize_r2rml_graph_source(&req.source, &req.target, req.force_full) + .await + .map_err(ServerError::Api)?; + + let response = IcebergMaterializeResponse { + source: req.source.clone(), + target: req.target.clone(), + from_snapshot_id: result.from_snapshot_id, + to_snapshot_id: result.to_snapshot_id, + incremental: result.incremental, + committed: result.committed, + rows_read: result.rows_read, + subjects_upserted: result.subjects_upserted, + }; + + tracing::info!( + status = "success", + source = %response.source, + target = %response.target, + to_snapshot_id = ?response.to_snapshot_id, + incremental = response.incremental, + committed = response.committed, + rows_read = response.rows_read, + subjects_upserted = response.subjects_upserted, + "iceberg materialize complete" + ); + Ok((StatusCode::OK, Json(response))) + } + .instrument(span) + .await +} + +/// Request body for `POST /v1/fluree/iceberg/track` and `/untrack`. +#[derive(Deserialize)] +pub struct IcebergTrackRequest { + /// Source graph source id to track. + pub source: String, + /// Target native ledger to keep materialized. + pub target: String, + /// How often the worker re-syncs this job, in seconds. Omit to use the + /// worker's default (30s). Ignored by `/untrack`. Must be > 0. + #[serde(default)] + pub poll_interval_secs: Option, +} + +/// Response for `POST /v1/fluree/iceberg/track`. +#[derive(Serialize)] +pub struct IcebergTrackResponse { + pub source: String, + pub target: String, + /// Whether the worker is now tracking this pair. + pub tracked: bool, + /// The effective poll interval for this job, in seconds. + pub poll_interval_secs: u64, + /// Number of jobs the worker is tracking. + pub tracked_jobs: usize, + /// Result of the immediate first materialize run on registration. + pub initial: IcebergMaterializeResponse, +} + +/// Register a `source → target` materialization tracking job and run an +/// immediate first sync. The worker then keeps the target fresh on its poll +/// interval (incremental when safe). +/// +/// POST /v1/fluree/iceberg/track +pub async fn iceberg_track(State(state): State>, request: Request) -> Response { + if state.config.server_role == ServerRole::Peer { + return forward_write_request(&state, request).await; + } + iceberg_track_local(state, request).await.into_response() +} + +async fn iceberg_track_local(state: Arc, request: Request) -> Result { + let _headers = FlureeHeaders::from_headers(request.headers())?; + let body_bytes = axum::body::to_bytes(request.into_body(), 1024 * 1024) + .await + .map_err(|e| ServerError::bad_request(format!("Failed to read body: {e}")))?; + let req: IcebergTrackRequest = serde_json::from_slice(&body_bytes) + .map_err(|e| ServerError::bad_request(format!("Invalid JSON: {e}")))?; + + if req.poll_interval_secs == Some(0) { + return Err(ServerError::bad_request( + "poll_interval_secs must be greater than 0", + )); + } + + let worker = state.materialize_worker.as_ref().ok_or_else(|| { + ServerError::bad_request("materialization tracking worker is not running on this node") + })?; + + let interval = worker.track( + &req.source, + &req.target, + req.poll_interval_secs.map(std::time::Duration::from_secs), + ); + + // Immediate first sync so the target is populated without waiting a cycle. + let result = state + .fluree + .materialize_r2rml_graph_source(&req.source, &req.target, false) + .await + .map_err(ServerError::Api)?; + + let response = IcebergTrackResponse { + source: req.source.clone(), + target: req.target.clone(), + tracked: true, + poll_interval_secs: interval.as_secs(), + tracked_jobs: worker.tracked_jobs().len(), + initial: IcebergMaterializeResponse { + source: req.source, + target: req.target, + from_snapshot_id: result.from_snapshot_id, + to_snapshot_id: result.to_snapshot_id, + incremental: result.incremental, + committed: result.committed, + rows_read: result.rows_read, + subjects_upserted: result.subjects_upserted, + }, + }; + Ok((StatusCode::OK, Json(response))) +} + +/// Stop tracking a `source → target` pair (leaves already-materialized data). +/// +/// POST /v1/fluree/iceberg/untrack +pub async fn iceberg_untrack(State(state): State>, request: Request) -> Response { + if state.config.server_role == ServerRole::Peer { + return forward_write_request(&state, request).await; + } + iceberg_untrack_local(state, request).await.into_response() +} + +async fn iceberg_untrack_local( + state: Arc, + request: Request, +) -> Result { + let _headers = FlureeHeaders::from_headers(request.headers())?; + let body_bytes = axum::body::to_bytes(request.into_body(), 1024 * 1024) + .await + .map_err(|e| ServerError::bad_request(format!("Failed to read body: {e}")))?; + let req: IcebergTrackRequest = serde_json::from_slice(&body_bytes) + .map_err(|e| ServerError::bad_request(format!("Invalid JSON: {e}")))?; + + let worker = state.materialize_worker.as_ref().ok_or_else(|| { + ServerError::bad_request("materialization tracking worker is not running on this node") + })?; + let removed = worker.untrack(&req.source, &req.target); + + Ok(( + StatusCode::OK, + Json(serde_json::json!({ + "source": req.source, + "target": req.target, + "removed": removed, + "tracked_jobs": worker.tracked_jobs().len(), + })), + )) +} + +/// Tracking-worker status: tracked jobs + cumulative stats. +/// +/// GET /v1/fluree/iceberg/tracking +pub async fn iceberg_tracking_status(State(state): State>) -> Response { + let Some(worker) = state.materialize_worker.as_ref() else { + return ( + StatusCode::OK, + Json(serde_json::json!({ "running": false, "jobs": [] })), + ) + .into_response(); + }; + let jobs: Vec<_> = worker + .job_infos() + .into_iter() + .map(|j| { + serde_json::json!({ + "source": j.source, + "target": j.target, + "poll_interval_secs": j.poll_interval_secs, + }) + }) + .collect(); + let stats = worker.stats(); + ( + StatusCode::OK, + Json(serde_json::json!({ + "running": true, + "jobs": jobs, + "stats": { + "polls": stats.polls, + "syncs_committed": stats.syncs_committed, + "syncs_noop": stats.syncs_noop, + "syncs_failed": stats.syncs_failed, + "tracked_jobs": stats.tracked_jobs, + } + })), + ) + .into_response() +} + fn build_iceberg_config(req: &IcebergMapRequest) -> Result { let mode = req.mode.to_lowercase(); let mut config = match mode.as_str() { diff --git a/fluree-db-server/src/routes/mod.rs b/fluree-db-server/src/routes/mod.rs index ca929f2967..cbb798969a 100644 --- a/fluree-db-server/src/routes/mod.rs +++ b/fluree-db-server/src/routes/mod.rs @@ -95,8 +95,11 @@ pub fn build_router(state: Arc) -> Router { ); #[cfg(feature = "iceberg")] - let v1_admin_protected_writes = - v1_admin_protected_writes.route("/iceberg/map", post(iceberg::iceberg_map)); + let v1_admin_protected_writes = v1_admin_protected_writes + .route("/iceberg/map", post(iceberg::iceberg_map)) + .route("/iceberg/materialize", post(iceberg::iceberg_materialize)) + .route("/iceberg/track", post(iceberg::iceberg_track)) + .route("/iceberg/untrack", post(iceberg::iceberg_untrack)); // Forward to the Raft leader (when running in Raft mode) before // admin-token auth: an out-of-date follower with stale credentials @@ -125,6 +128,11 @@ pub fn build_router(state: Arc) -> Router { // `state.import_jobs` map (each node owns the jobs it minted). .route("/import-upload/:import_id", get(import::import_status)); + // Materialization tracking-worker status — reads this node's worker state. + #[cfg(feature = "iceberg")] + let v1_admin_protected_reads = + v1_admin_protected_reads.route("/iceberg/tracking", get(iceberg::iceberg_tracking_status)); + let v1_admin_protected_reads = v1_admin_protected_reads .layer(middleware::from_fn_with_state( state.clone(), diff --git a/fluree-db-server/src/state.rs b/fluree-db-server/src/state.rs index 8a832ed208..0b75134678 100644 --- a/fluree-db-server/src/state.rs +++ b/fluree-db-server/src/state.rs @@ -93,6 +93,16 @@ pub struct AppState { /// Aborted on drop so the `Arc` doesn't outlive the server. cache_stats_handle: Option>, + /// Handle to the R2RML/Iceberg materialization tracking worker for + /// registering `source → target` jobs. `None` on peer-mode nodes (which + /// forward writes) or when the worker failed to start. + #[cfg(feature = "iceberg")] + pub materialize_worker: Option, + + /// Join handle for the materialization worker task; aborted on drop. + #[cfg(feature = "iceberg")] + materialize_worker_task: Option>, + /// Registry of in-flight negotiated-upload import jobs (reference impl of /// the presigned `.flpack` upload flow). Empty/unused unless /// `config.import_presign_enabled`. @@ -219,6 +229,22 @@ impl AppState { index_config.clone(), )); + // Spawn the R2RML/Iceberg materialization tracking worker on write + // nodes. Peers forward writes elsewhere, so they don't run it. (In Raft + // mode the worker currently runs on every node and writes directly via + // `Fluree::upsert`; gating it to the leader is a follow-up — single-node + // and peer deployments are correct today.) + #[cfg(feature = "iceberg")] + let (materialize_worker, materialize_worker_task) = + if config.server_role != ServerRole::Peer { + let worker = fluree_db_api::MaterializeTrackingWorker::new(Arc::clone(&fluree)); + let handle = worker.handle(); + let task = tokio::spawn(worker.run()); + (Some(handle), Some(task)) + } else { + (None, None) + }; + Ok(Self { fluree, config, @@ -236,6 +262,10 @@ impl AppState { refresh_counter: AtomicU64::new(0), query_refresh_last_checked: DashMap::new(), cache_stats_handle: Some(cache_stats_handle), + #[cfg(feature = "iceberg")] + materialize_worker, + #[cfg(feature = "iceberg")] + materialize_worker_task, import_jobs: Arc::new(crate::import_jobs::ImportJobs::default()), }) } @@ -300,6 +330,10 @@ impl Drop for AppState { if let Some(handle) = self.cache_stats_handle.take() { handle.abort(); } + #[cfg(feature = "iceberg")] + if let Some(task) = self.materialize_worker_task.take() { + task.abort(); + } } } From 6fcfda2c33227e51f5e9d9abb778016e3a04a4a7 Mon Sep 17 00:00:00 2001 From: Christophe Di Prima Date: Wed, 1 Jul 2026 09:52:54 +0200 Subject: [PATCH 4/5] feat(iceberg): tombstone deletion + latest-by-key for R2RML materialization Extend materialization with change-data-capture semantics so a materialized native ledger tracks source updates and deletes, not just inserts. - DeleteConvention { column, deleted_values: Vec> } on IcebergGsConfig: a row is a delete when its column value is in deleted_values; a null entry matches a NULL column (Debezium null-payload). Threaded through the create builders + /iceberg/map route; validated at graph-source creation. - order_by column enables latest-by-key: per subject the highest-ordered row wins (value-orderable int/date/timestamp only, enforced) with a whole-subject replace that clears fields dropped in a newer revision; a tombstone retracts the whole subject via a typed-@id wildcard update. Two ordered commits keep the watermark advancing only in the upsert (crash/failure self-heals next poll). - Per-(source,table) watermark with an injective subject; each table is read once with its own (from,to] window. Dual-mode: with neither delete nor order_by set the pass is the legacy additive merge (unchanged). Fails loud on a non-orderable order_by column or multiple triples maps per table. - term.rs exposes column_string / batch_has_column / column_sort_key / column_is_orderable. subjects_retracted on MaterializeResult / route / worker. - Docs: a "Materialization" guide in docs/graph-sources/iceberg.md plus the materialize/track/untrack/tracking endpoints in docs/api/endpoints.md. - Tests: latest-by-key ordering + retraction-doc-shape unit tests, and an end-to-end integration test proving retraction actually deletes (with a control proving the earlier bare-string form was a silent no-op). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/api/endpoints.md | 75 ++ docs/graph-sources/iceberg.md | 141 ++++ fluree-db-api/src/graph_source/config.rs | 55 ++ fluree-db-api/src/graph_source/r2rml.rs | 27 +- .../src/graph_source/r2rml_materialize.rs | 657 +++++++++++++++--- fluree-db-api/src/lib.rs | 3 + fluree-db-api/src/materialize_worker.rs | 1 + fluree-db-api/tests/grp_graphsource.rs | 2 + fluree-db-api/tests/it_materialize_retract.rs | 97 +++ fluree-db-iceberg/src/config.rs | 163 +++++ fluree-db-iceberg/src/lib.rs | 4 +- fluree-db-r2rml/src/materialize/mod.rs | 1 + fluree-db-r2rml/src/materialize/term.rs | 72 ++ fluree-db-server/src/routes/iceberg.rs | 27 + 14 files changed, 1212 insertions(+), 113 deletions(-) create mode 100644 fluree-db-api/tests/it_materialize_retract.rs diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index 4c65608ba5..42d1554bc0 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -2547,6 +2547,9 @@ POST http://localhost:8090/v1/fluree/iceberg/map | `warehouse` | string | Warehouse identifier | | `no_vended_credentials` | bool | Disable vended credentials | | `s3_region`, `s3_endpoint`, `s3_path_style` | | S3 overrides for `direct` mode | +| `order_by` | string | Latest-by-key ordering column for materialization (int/date/timestamp) | +| `delete_column` | string | Column that marks a row as a delete (tombstone) during materialization | +| `delete_values` | (string\|null)[] | Values of `delete_column` that mean "deleted"; a `null` entry matches a NULL column (null-payload delete). Required when `delete_column` is set. | **Response:** @@ -2570,6 +2573,78 @@ POST http://localhost:8090/v1/fluree/iceberg/map See also the CLI wrapper: [fluree iceberg map](../cli/iceberg.md). +### POST {api_base_url}/iceberg/materialize + +Materialize a graph source into a native ledger (so BM25 / vector / reasoning can run over it). Reads incrementally from a per-table watermark persisted in the target, or fully with `force_full`. Admin-protected; `iceberg` feature only. See [Materialization](../graph-sources/iceberg.md#materialization-into-a-native-ledger). + +**Request Body:** + +```json +{ "source": "orders:main", "target": "orders-native:main", "force_full": false } +``` + +| Field | Type | Description | +|-------|------|-------------| +| `source` | string | Source graph source id to read (required) | +| `target` | string | Target native ledger to write (required; created if absent) | +| `force_full` | bool | Ignore the watermark and re-read the whole table (default `false`) | + +**Response:** + +```json +{ + "source": "orders:main", + "target": "orders-native:main", + "from_snapshot_id": null, + "to_snapshot_id": 5648190075564901028, + "incremental": false, + "committed": true, + "rows_read": 1200, + "subjects_upserted": 1200, + "subjects_retracted": 0 +} +``` + +`committed: false` means a no-delta poll (nothing changed since the watermark). `from_snapshot_id`/`to_snapshot_id` are populated for single-table mappings (multi-table watermarks live per-table in the target ledger). + +**Status Codes:** `200 OK`; `400` invalid request / config; `401/403` admin auth; `500` scan or commit failure. + +### POST {api_base_url}/iceberg/track + +Register a `source → target` materialization job with the in-process tracking worker, run an immediate first sync, and keep the target fresh on a timer. Admin-protected; `iceberg` feature only. + +**Request Body:** + +```json +{ "source": "orders:main", "target": "orders-native:main", "poll_interval_secs": 300 } +``` + +| Field | Type | Description | +|-------|------|-------------| +| `source` | string | Graph-source id to read. Required. | +| `target` | string | Native ledger to keep materialized. Required. | +| `poll_interval_secs` | number | This job's own re-sync cadence, in seconds. Optional — defaults to the worker's configured interval (30s). Must be > 0. Each tracked job runs on its own timer. | + +**Response:** `{ "source", "target", "tracked": true, "poll_interval_secs": 300, "tracked_jobs": 1, "initial": { …materialize response… } }` + +Tracked jobs are held in memory (re-issue `track` after a restart; the watermark in the target ledger means it resumes incrementally). The worker runs on write nodes only. + +### POST {api_base_url}/iceberg/untrack + +Stop tracking a `source → target` pair (already-materialized data is left in place). Body: `{ "source", "target" }`. Response: `{ "source", "target", "removed": true, "tracked_jobs": 0 }`. + +### GET {api_base_url}/iceberg/tracking + +List the tracking worker's jobs and cumulative stats. + +```json +{ + "running": true, + "jobs": [ { "source": "orders:main", "target": "orders-native:main", "poll_interval_secs": 300 } ], + "stats": { "polls": 12, "syncs_committed": 1, "syncs_noop": 11, "syncs_failed": 0, "tracked_jobs": 1 } +} +``` + ## Admin Endpoints ### POST /reindex diff --git a/docs/graph-sources/iceberg.md b/docs/graph-sources/iceberg.md index a26595ef65..eb626f4d6d 100644 --- a/docs/graph-sources/iceberg.md +++ b/docs/graph-sources/iceberg.md @@ -315,6 +315,147 @@ ORDER BY DESC(?date) LIMIT 100 ``` +## Materialization (into a native ledger) + +Querying a graph source reads the Iceberg table on the fly. Native Fluree +features — **BM25 full-text search, vector / RAG, and reasoning** — operate only +on facts committed to a *native* ledger. To use those over an Iceberg table, +**materialize** it: Fluree expands the R2RML mapping over the source rows and +`upsert`s the resulting triples into a target native ledger, which you can then +index and reason over like any other ledger. + +> Requires the `iceberg` feature. The endpoints are admin-protected (send the +> admin Bearer token when one is configured). + +### One-shot materialize + +`POST {api_base_url}/iceberg/materialize` reads the source and writes it into the +target ledger (created if it does not exist): + +```bash +curl -X POST http://localhost:8090/v1/fluree/iceberg/materialize \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -d '{ "source": "orders:main", "target": "orders-native:main" }' +``` + +```json +{ + "source": "orders:main", + "target": "orders-native:main", + "from_snapshot_id": null, + "to_snapshot_id": 5648190075564901028, + "incremental": false, + "committed": true, + "rows_read": 1200, + "subjects_upserted": 1200, + "subjects_retracted": 0 +} +``` + +The materialized snapshot id is persisted as a **watermark** inside the target +ledger (one per source table), so re-running resumes **incrementally** — only +the rows added since the last run are read. You track nothing; just call it +again. A run with no new data commits nothing and returns `committed: false`. +Pass `"force_full": true` to ignore the watermark and re-read the whole table. + +Incremental reads apply when the source's snapshot window is append- or +compaction-only; an `overwrite`/`delete` snapshot, or expired history, falls back +to a full re-read automatically. + +### Tracking (keep the target fresh automatically) + +`POST {api_base_url}/iceberg/track` registers a `source → target` job with the +in-process tracking worker, runs an immediate first sync, and then refreshes the +target on a timer (default every 30s): + +```bash +curl -X POST http://localhost:8090/v1/fluree/iceberg/track \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -d '{ "source": "orders:main", "target": "orders-native:main" }' +``` + +- `POST {api_base_url}/iceberg/untrack` — stop tracking (leaves materialized data in place). +- `GET {api_base_url}/iceberg/tracking` — list tracked jobs and worker stats. + +The worker runs on write nodes (not peers). Tracked jobs are held in memory, so +after a restart re-issue `track` — the watermark in the target ledger means it +resumes incrementally rather than re-reading everything. + +### Change data capture: updates and deletes (latest-by-key) + +By default materialization is **additive** (it inserts/updates triples; it never +removes them). For a change-data-capture source — an append-only log where each +change is a new row and a delete is a **tombstone row** — configure two options +so Fluree applies *latest-by-key* semantics that match a +`ROW_NUMBER() OVER (PARTITION BY id ORDER BY DESC) = 1` view: + +- **`order_by`** — a column that orders a key's revisions (e.g. an event + timestamp or offset). Must be an **integer / date / timestamp** column. The + latest row per subject wins, and a whole-subject *replace* clears fields that + were dropped in the newer revision. +- **`delete_column`** + **`delete_values`** — how a row is recognized as a + delete. `delete_values` lists the `delete_column` values that mean "deleted"; + a `null` entry matches a NULL column (the Debezium null-payload convention). + When the latest row for a key is a tombstone, the **entire subject** (all its + triples) is retracted. + +Two ways to encode a delete: + +```bash +# (a) value-match: an op column carries "d" on a delete (Debezium-style). +curl -X POST http://localhost:8090/v1/fluree/iceberg/map \ + -H 'Content-Type: application/json' \ + -d '{ "name": "orders", "mode": "direct", + "table_location": "s3://my-bucket/warehouse/sales/orders", + "r2rml": "...", "r2rml_type": "text/turtle", + "order_by": "event_timestamp", + "delete_column": "_op", "delete_values": ["d", "delete"] }' + +# (b) null-payload: a delete row has the key set but content columns null; +# pick a column always set on a live row but null on a delete, and list null. +curl -X POST http://localhost:8090/v1/fluree/iceberg/map \ + -H 'Content-Type: application/json' \ + -d '{ "name": "orders", "mode": "direct", + "table_location": "s3://my-bucket/warehouse/sales/orders", + "r2rml": "...", "r2rml_type": "text/turtle", + "order_by": "event_timestamp", + "delete_column": "status", "delete_values": [null] }' + +# (combine both: a "d" op value OR a null column both mean delete) +# "delete_column": "_op", "delete_values": ["d", null] +``` + +These live in the graph source's stored config (`IcebergGsConfig.delete` and +`order_by`); set them once at `iceberg map` time. A delete removes the **whole +entity**, not individual columns — a `null` in an ordinary column of a *live* row +just clears that one predicate. + +### Assumptions and limitations + +Latest-by-key mode (i.e. when `order_by` and/or a delete convention is set) +assumes the source matches the append-only, full-image CDC shape these features +target. The materializer enforces what it can and documents the rest: + +- **One complete row per subject revision.** Each row is a full snapshot of its + subject. A source that assembles one subject across multiple rows (e.g. an + unpivoted join table) is not supported in latest-by-key mode — use additive + mode (omit `order_by`/`delete`) or a one-row-per-subject view. +- **One triples map per logical table** (enforced — multiple would clobber under + whole-subject replace). +- **`order_by` must be populated and value-orderable** (int/date/timestamp, + enforced). A row with a null ordering value sorts as oldest. +- **The target ledger is dedicated to one source.** Whole-subject retraction owns + the subject; don't mix other sources or hand-written data about the same IRIs + into the same target. +- **Deletes must be expressed as tombstone rows.** A key that simply stops + appearing — with no tombstone — is not reconciled (a set-difference pass is a + possible future addition). +- Retract and re-assert are two ordered commits; the watermark advances only in + the second, so an interrupted run re-materializes the same window on the next + pass (self-healing). + ## Partition Pruning Iceberg's partition pruning optimizes queries: diff --git a/fluree-db-api/src/graph_source/config.rs b/fluree-db-api/src/graph_source/config.rs index 3815663d71..3886e925f7 100644 --- a/fluree-db-api/src/graph_source/config.rs +++ b/fluree-db-api/src/graph_source/config.rs @@ -411,6 +411,15 @@ pub struct IcebergCreateConfig { /// Use path-style S3 URLs pub s3_path_style: bool, + + /// Optional tombstone/delete convention for materialization (which source + /// column + value(s)/null mark a row as a delete). `None` => additive + /// materialization (no retraction). + pub delete_convention: Option, + + /// Optional ordering column for latest-by-key materialization (e.g. an event + /// timestamp or offset). `None` => last-in-scan-order wins. + pub order_by: Option, } /// How the Iceberg catalog is accessed. @@ -464,6 +473,8 @@ impl IcebergCreateConfig { s3_region: None, s3_endpoint: None, s3_path_style: false, + delete_convention: None, + order_by: None, } } @@ -478,6 +489,8 @@ impl IcebergCreateConfig { s3_region: None, s3_endpoint: None, s3_path_style: false, + delete_convention: None, + order_by: None, } } @@ -615,6 +628,21 @@ impl IcebergCreateConfig { self } + /// Set the tombstone/delete convention used during materialization. + pub fn with_delete_convention( + mut self, + convention: fluree_db_iceberg::DeleteConvention, + ) -> Self { + self.delete_convention = Some(convention); + self + } + + /// Set the ordering column for latest-by-key materialization. + pub fn with_order_by(mut self, column: impl Into) -> Self { + self.order_by = Some(column.into()); + self + } + /// Get the effective branch name. pub fn effective_branch(&self) -> &str { self.branch.as_deref().unwrap_or(DEFAULT_BRANCH) @@ -675,6 +703,8 @@ impl IcebergCreateConfig { s3_path_style: self.s3_path_style, }, mapping: None, + delete: self.delete_convention.clone(), + order_by: self.order_by.clone(), }, CatalogMode::Direct { table_location } => IcebergGsConfig { catalog: CatalogConfig::direct(table_location), @@ -686,6 +716,8 @@ impl IcebergCreateConfig { s3_path_style: self.s3_path_style, }, mapping: None, + delete: self.delete_convention.clone(), + order_by: self.order_by.clone(), }, } } @@ -729,6 +761,14 @@ impl IcebergCreateConfig { } } + // Validate the tombstone/delete convention at creation time rather than + // deferring to the first materialize scan. + if let Some(delete) = &self.delete_convention { + delete + .validate() + .map_err(|e| crate::ApiError::config(format!("Invalid delete convention: {e}")))?; + } + Ok(()) } @@ -901,6 +941,21 @@ impl R2rmlCreateConfig { self } + /// Set the tombstone/delete convention used during materialization. + pub fn with_delete_convention( + mut self, + convention: fluree_db_iceberg::DeleteConvention, + ) -> Self { + self.iceberg = self.iceberg.with_delete_convention(convention); + self + } + + /// Set the ordering column for latest-by-key materialization. + pub fn with_order_by(mut self, column: impl Into) -> Self { + self.iceberg = self.iceberg.with_order_by(column); + self + } + /// Get the graph source ID (name:branch). pub fn graph_source_id(&self) -> String { self.iceberg.graph_source_id() diff --git a/fluree-db-api/src/graph_source/r2rml.rs b/fluree-db-api/src/graph_source/r2rml.rs index dfb06247fa..4612244fcb 100644 --- a/fluree-db-api/src/graph_source/r2rml.rs +++ b/fluree-db-api/src/graph_source/r2rml.rs @@ -16,7 +16,7 @@ use fluree_db_iceberg::{ io::{ColumnBatch, S3IcebergStorage, SendIcebergStorage, SendParquetReader}, metadata::TableMetadata, scan::{ComparisonOp, Expression, FileScanTask, LiteralValue, ScanConfig, SendScanPlanner}, - IcebergGsConfig, + DeleteConvention, IcebergGsConfig, }; use fluree_db_nameservice::GraphSourceType; use fluree_db_query::error::{QueryError, Result as QueryResult}; @@ -688,6 +688,31 @@ impl<'a> FlureeR2rmlProvider<'a> { Ok(metadata.current_snapshot().map(|s| s.snapshot_id)) } + /// The source graph source's materialization options from the persisted + /// `IcebergGsConfig`: the optional tombstone/delete convention and the + /// optional latest-by-key ordering column. Both `None` means additive, + /// scan-order materialization (legacy behavior). + pub async fn materialize_options( + &self, + graph_source_id: &str, + ) -> QueryResult<(Option, Option)> { + let record = self + .fluree + .nameservice() + .lookup_graph_source(graph_source_id) + .await + .map_err(|e| QueryError::Internal(format!("Nameservice error: {e}")))? + .ok_or_else(|| { + QueryError::InvalidQuery(format!("Graph source '{graph_source_id}' not found")) + })?; + let config = IcebergGsConfig::from_json(&record.config).map_err(|e| { + QueryError::Internal(format!( + "Failed to parse Iceberg graph source config for '{graph_source_id}': {e}" + )) + })?; + Ok((config.delete, config.order_by)) + } + /// Scan only the data files ADDED in the snapshot window /// `(from_snapshot_id, to_snapshot_id]` (append-only incremental). /// diff --git a/fluree-db-api/src/graph_source/r2rml_materialize.rs b/fluree-db-api/src/graph_source/r2rml_materialize.rs index 9275fe3e80..dc4b6a519b 100644 --- a/fluree-db-api/src/graph_source/r2rml_materialize.rs +++ b/fluree-db-api/src/graph_source/r2rml_materialize.rs @@ -8,24 +8,55 @@ //! terms exactly as the query path does, group them per subject, and `upsert` //! them into a target ledger. //! -//! Refreshes are incremental when safe. The caller passes the last materialized -//! snapshot id as `from_snapshot_id`; if the source's `(from, to]` window is -//! append/compaction-only (see -//! [`window_is_incremental_safe`](fluree_db_iceberg::metadata::TableMetadata::window_is_incremental_safe)), -//! only the *added* rows are read and upserted. The returned `to_snapshot_id` is -//! the new watermark for the caller to persist — watermark storage is left to -//! the caller (the tracking worker) so this stays free of storage assumptions. +//! Refreshes are incremental when safe. A per-(source, table) watermark (the +//! last materialized snapshot id) is persisted as a triple in the *target* +//! ledger and read back automatically, so callers track nothing. When the +//! source's `(from, to]` window is append/compaction-only (see +//! [`window_is_incremental_safe`](fluree_db_iceberg::metadata::TableMetadata::window_is_incremental_safe)) +//! only the *added* rows are read; otherwise the live table is full-read. +//! +//! With a `delete` convention + `order_by` column configured on the source, the +//! pass is **latest-by-key**: per subject the highest-ordered row wins, a +//! whole-subject *replace* clears fields dropped in a newer revision, and a +//! tombstone row retracts the subject — matching a `ROW_NUMBER() … ORDER BY +//! DESC` view. +//! +//! # Assumptions (latest-by-key mode) +//! +//! These hold for the append-only, full-image CDC sinks this targets (e.g. +//! Debezium → Iceberg) and are enforced where cheap, else documented: +//! - **One complete row per subject revision.** Each source row is a full +//! snapshot of its subject (whole-row replace assumes this). A source where a +//! subject is *assembled across multiple rows* (e.g. an unpivoted join table) +//! is not supported in latest-by-key mode — use legacy mode (no `delete`/ +//! `order_by`) or a one-row-per-subject view. **One triples map per logical +//! table** is required and enforced (multiple would clobber under replace). +//! - **`order_by` is a populated, value-orderable column** (integer / date / +//! timestamp — enforced; float/decimal/string are rejected). A row with a NULL +//! ordering value sorts as oldest, so the ordering column should be present on +//! every row (CDC event timestamps are). +//! - **The target ledger is dedicated to this source** — whole-subject +//! retraction owns the subject; mixing other sources or hand-written data +//! about the same IRIs into the target is unsupported. +//! - **Two ordered commits, not one** (a combined delete+insert silently drops +//! the grounded insert when the delete binds zero rows). The watermark advances +//! only in the second (upsert) commit, so a crash — or a failed upsert — leaves +//! the watermark un-advanced and the next poll re-materializes the same window +//! (self-healing; a failed upsert leaves the just-retracted subjects missing +//! only until that next poll). //! //! This module is only available with the `iceberg` feature. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use serde_json::{json, Map, Value as JsonValue}; use fluree_db_iceberg::io::ColumnBatch; +use fluree_db_iceberg::DeleteConvention; use fluree_db_query::r2rml::R2rmlProvider; use fluree_db_r2rml::mapping::TriplesMap; use fluree_db_r2rml::materialize::{ + batch_has_column, column_is_orderable, column_sort_key, column_string, materialize_object_from_batch, materialize_subject_from_batch, RdfTerm, }; use fluree_vocab::UnresolvedDatatypeConstraint; @@ -33,15 +64,18 @@ use fluree_vocab::UnresolvedDatatypeConstraint; use crate::graph_source::FlureeR2rmlProvider; use crate::{ApiError, Fluree, Result}; -/// Subject-IRI prefix for the per-source materialization watermark stored in the -/// target ledger. The full subject is `{PREFIX}{source_graph_source_id}` so one -/// target ledger can track several sources independently. +/// Subject-IRI prefix for the per-(source, table) materialization watermark +/// stored in the target ledger. The full subject is +/// `{PREFIX}{source_graph_source_id}:{table_name}` so one target ledger can +/// track several sources — and each source's tables — independently. const WATERMARK_SUBJECT_PREFIX: &str = "urn:fluree:materialize-state:"; /// Predicate holding the last materialized source snapshot id (stored as a /// string to preserve full i64 precision for 19-digit snapshot ids). const WATERMARK_SNAPSHOT_PRED: &str = "urn:fluree:materialize#lastSnapshotId"; /// Predicate recording which source the watermark belongs to (informational). const WATERMARK_SOURCE_PRED: &str = "urn:fluree:materialize#source"; +/// Predicate recording which source table the watermark belongs to. +const WATERMARK_TABLE_PRED: &str = "urn:fluree:materialize#table"; /// Outcome of one materialization pass. #[derive(Debug, Clone)] @@ -62,35 +96,40 @@ pub struct MaterializeResult { pub committed: bool, /// Number of source rows read across all batches. pub rows_read: usize, - /// Number of distinct subject nodes upserted. + /// Number of distinct subject nodes upserted (live rows). pub subjects_upserted: usize, + /// Number of distinct subjects retracted (tombstone rows whose final state in + /// this pass was a delete). Always 0 when no delete convention is configured + /// or on the first materialization of a not-yet-existing target. + pub subjects_retracted: usize, } impl Fluree { /// Materialize an R2RML / Iceberg graph source into a native ledger. /// - /// Resolves the starting point from the **watermark persisted in the target - /// ledger** (the last materialized source snapshot id) — unless `force_full` - /// is set, which ignores it and re-reads the whole live table. Reads the - /// source (incrementally when the `(from, to]` window is append/compaction - /// only, else full), expands each row through the source's compiled R2RML - /// mapping into RDF terms, and `upsert`s one JSON-LD node per subject into - /// `target_ledger_id` (created if absent). The new watermark is written in - /// the **same** `upsert` commit, so it advances atomically with the data and - /// the operation is idempotent. Callers track nothing; a bare re-run resumes - /// incrementally. This makes the call safe to invoke on a timer (the - /// tracking worker does exactly that). - /// - /// A no-delta poll (`from == to`) commits nothing and returns - /// `committed = false`. + /// Each logical table in the mapping is materialized independently against + /// its **own** per-(source, table) watermark, persisted as a triple in the + /// target ledger and read back automatically (unless `force_full` ignores it + /// and re-reads the whole live table). A table reads incrementally when its + /// `(from, to]` window is append/compaction-only, else full. Callers track + /// nothing; a bare re-run resumes incrementally — safe to invoke on a timer + /// (the tracking worker does exactly that). A no-delta poll commits nothing + /// and returns `committed = false`. /// - /// **Deletion safety.** An added-files scan captures inserts (and in-place - /// updates land via `upsert`), but it cannot see row *deletions*. The window - /// check therefore forces a full re-read whenever an `overwrite`/`delete` - /// snapshot appears, so deletes are never silently missed — while routine - /// appends and periodic compaction stay on the cheap incremental path. A - /// full re-read still only `upsert`s present rows; reconciling subjects that - /// vanished entirely (set-difference retraction) is a separate follow-up. + /// **Latest-by-key + deletions.** With an `order_by` column configured, the + /// rows of each subject in the window are ranked and the **latest** row wins; + /// without one, the last row in scan order wins. The winning row is applied + /// as a *whole-subject replace*: every subject seen in the window is + /// retracted (clearing fields dropped in a newer revision), then the live + /// nodes are re-asserted with the advanced watermarks. A row classified as a + /// tombstone by the [`DeleteConvention`] (`IcebergGsConfig.delete`) retracts + /// its subject and is not re-asserted. All subject IRIs come from the same + /// subject materializer (parity with the query path). The two commits are + /// ordered (retract, then upsert+watermark) so the watermark advances only in + /// the upsert — a crash between them re-reads the same window on the next run. + /// The target ledger is assumed dedicated to this source (whole-subject + /// retraction owns the subject). With neither `order_by` nor `delete` + /// configured the pass is additive per-predicate (legacy behavior). pub async fn materialize_r2rml_graph_source( &self, source_graph_source_id: &str, @@ -99,7 +138,8 @@ impl Fluree { ) -> Result { let provider = FlureeR2rmlProvider::new(self); - // 1. Compiled R2RML mapping (subject / predicate / object maps). + // 1. Compiled R2RML mapping (subject / predicate / object maps) and the + // materialization options (delete convention + latest-by-key ordering). let mapping = provider .compiled_mapping(source_graph_source_id, None) .await?; @@ -108,56 +148,132 @@ impl Fluree { "Graph source '{source_graph_source_id}' has no R2RML triples maps" ))); } + let (delete_convention, order_by) = + provider.materialize_options(source_graph_source_id).await?; + // Latest-by-key (whole-subject replace + tombstone deletes) is enabled by + // configuring a delete convention and/or an ordering column. With neither, + // the pass is the legacy additive merge (a subject may span multiple rows). + let latest_by_key = delete_convention.is_some() || order_by.is_some(); + let target_existed = self.ledger_exists(target_ledger_id).await?; + + // 2. Group triples maps by their logical table (deterministic order), then + // read each table ONCE against its OWN per-(source,table) watermark and + // `(from, to]` window, applying ALL of that table's triples maps to each + // row. Classify rows into a latest-by-key accumulator. + let mut tables: BTreeMap<&str, Vec<&TriplesMap>> = BTreeMap::new(); + for tm in mapping.triples_maps.values() { + if let Some(table_name) = tm.table_name() { + tables.entry(table_name).or_default().push(tm); + } + } + // Latest-by-key requires one triples map per logical table: multiple + // triples maps over the same table that emit the same subject would + // clobber each other's predicates under whole-row replace. Fail loud + // rather than silently drop triples. (Legacy additive mode merges, so it + // is fine.) + if latest_by_key { + if let Some((table_name, tms)) = tables.iter().find(|(_, tms)| tms.len() > 1) { + return Err(ApiError::Config(format!( + "latest-by-key materialization (delete/order_by configured) supports one \ + triples map per logical table, but '{table_name}' has {} — split them into \ + separate tables/sources or remove delete/order_by", + tms.len() + ))); + } + } - // 2. Resolve the starting watermark from the target ledger (unless the - // caller forces a full re-read). - let from_snapshot_id = if force_full { - None - } else { - self.materialize_watermark(target_ledger_id, source_graph_source_id) - .await? - }; - - // 3. Read source rows (incremental when safe) per logical table and - // aggregate all triples per subject IRI into one JSON-LD node. The - // BTreeMap keeps subjects in a stable order (deterministic txn). - let mut subjects: BTreeMap = BTreeMap::new(); + let mut accum = MaterializeAccum::default(); let mut rows_read = 0usize; - let mut to_snapshot_id: Option = None; - // The pass is incremental only if *every* table's scan was incremental - // (and we resolved a watermark to begin with). - let mut incremental_all = from_snapshot_id.is_some(); + let mut incremental_all = true; let mut any_table = false; + let mut all_watermarks_unchanged = true; + // (table, from-snapshot, advanced to-snapshot) per source table. + let mut table_watermarks: Vec<(String, Option, i64)> = Vec::new(); - for tm in mapping.triples_maps.values() { - let Some(table_name) = tm.table_name() else { - // Subject-only / constant-IRI maps with no logical table. - continue; + for (table_name, tms) in &tables { + let from_t = if force_full || !target_existed { + None + } else { + self.materialize_watermark(target_ledger_id, source_graph_source_id, table_name) + .await? }; - any_table = true; let (to_id, incremental, batches) = provider - .scan_for_materialize(source_graph_source_id, table_name, &[], from_snapshot_id) + .scan_for_materialize(source_graph_source_id, table_name, &[], from_t) .await?; - to_snapshot_id = to_id; - incremental_all = incremental_all && incremental; + // Only count a table as contributing once it actually has a snapshot. + if let Some(to) = to_id { + any_table = true; + incremental_all = incremental_all && incremental; + if Some(to) != from_t { + all_watermarks_unchanged = false; + } + table_watermarks.push(((*table_name).to_string(), from_t, to)); + } for batch in &batches { + // Configured marker / ordering columns must exist (and the + // ordering column must be a value-orderable type); otherwise a + // null-match convention would treat every row as a tombstone and + // ordering would silently mis-sort. + if let Some(conv) = delete_convention.as_ref() { + if !batch_has_column(batch, &conv.column) { + return Err(ApiError::Config(format!( + "delete.column '{}' not found in source table '{table_name}'", + conv.column + ))); + } + } + if let Some(col) = order_by.as_deref() { + if !batch_has_column(batch, col) { + return Err(ApiError::Config(format!( + "order_by column '{col}' not found in source table '{table_name}'" + ))); + } + if !column_is_orderable(batch, col) { + return Err(ApiError::Config(format!( + "order_by column '{col}' in source table '{table_name}' must be an \ + integer, date, or timestamp type for value-correct latest-by-key \ + ordering" + ))); + } + } for row in 0..batch.num_rows { rows_read += 1; - materialize_row_into(tm, batch, row, &mut subjects)?; + for tm in tms { + materialize_row_into( + tm, + batch, + row, + delete_convention.as_ref(), + order_by.as_deref(), + latest_by_key, + &mut accum, + )?; + } } } } let incremental = any_table && incremental_all; - let subjects_upserted = subjects.len(); + // Surface from/to on the result only for a single-table mapping (one + // materialized table); multi-table watermarks live per-table in the ledger. + let (from_snapshot_id, to_snapshot_id) = if table_watermarks.len() == 1 { + (table_watermarks[0].1, Some(table_watermarks[0].2)) + } else { + (None, None) + }; + + // 3. Finalize the latest-by-key state: live nodes to (re)assert and the + // set of subjects whose latest row is a tombstone (deletions). + let (live, deletions) = accum.finalize(); + let subjects_upserted = live.len(); + // Deletions are only actually applied when the target already existed + // (retraction is skipped on a brand-new target — nothing to retract). + let subjects_retracted = if target_existed { deletions.len() } else { 0 }; - // 4. Decide whether anything needs committing. Nothing changed when the - // source has no snapshot, or the watermark already equals `to` and no - // rows were produced (a no-delta poll) — skip the commit entirely. - let watermark_unchanged = to_snapshot_id.is_none() || to_snapshot_id == from_snapshot_id; - if subjects.is_empty() && watermark_unchanged { + // No-delta short-circuit: nothing read and no watermark advanced. + if live.is_empty() && deletions.is_empty() && all_watermarks_unchanged { return Ok(MaterializeResult { to_snapshot_id, from_snapshot_id, @@ -165,25 +281,46 @@ impl Fluree { committed: false, rows_read, subjects_upserted: 0, + subjects_retracted: 0, }); } - // 5. Build the JSON-LD insert array: one node per subject, plus the - // advanced watermark node (committed atomically with the data). - let mut nodes: Vec = - subjects.into_values().map(SubjectNode::into_json).collect(); - if let Some(to) = to_snapshot_id { - nodes.push(watermark_node(source_graph_source_id, to)); - } - let doc = JsonValue::Array(nodes); + // 4. In latest-by-key mode, whole-subject REPLACE: retract every subject + // seen in this window (live OR tombstone) so a live revision that + // dropped a field clears the stale value and a tombstone is removed; + // then re-assert the live nodes + advance the per-table watermarks. Two + // ordered commits keep the watermark advancing ONLY in the upsert + // (crash-safe: a crash between them re-reads the same window next run). + // Retraction is skipped on a brand-new target (nothing to retract) and + // in legacy additive mode (per-predicate upsert suffices; a subject may + // legitimately span rows we must not clobber). + let retract_all: BTreeSet = if latest_by_key { + live.keys() + .cloned() + .chain(deletions.iter().cloned()) + .collect() + } else { + BTreeSet::new() + }; - // 6. Load-or-create the target ledger and upsert. - let ledger = if self.ledger_exists(target_ledger_id).await? { + let mut ledger = if target_existed { self.ledger(target_ledger_id).await? } else { self.create_ledger(target_ledger_id).await? }; - self.upsert(ledger, &doc).await?; + + if target_existed && !retract_all.is_empty() { + ledger = self + .update(ledger, &build_retract_doc(&retract_all)) + .await? + .ledger; + } + + let mut nodes: Vec = live.into_values().map(SubjectNode::into_json).collect(); + for (table, _from, to) in &table_watermarks { + nodes.push(watermark_node(source_graph_source_id, table, *to)); + } + self.upsert(ledger, &JsonValue::Array(nodes)).await?; Ok(MaterializeResult { to_snapshot_id, @@ -192,23 +329,26 @@ impl Fluree { committed: true, rows_read, subjects_upserted, + subjects_retracted, }) } - /// Read the materialization watermark (last materialized source snapshot id) - /// for `source_graph_source_id` from the target ledger. Returns `None` if the - /// target ledger does not exist yet or carries no watermark for that source. + /// Read the per-table materialization watermark (last materialized source + /// snapshot id) for `(source_graph_source_id, table_name)` from the target + /// ledger. Returns `None` if the target ledger does not exist yet or carries + /// no watermark for that source table. pub async fn materialize_watermark( &self, target_ledger_id: &str, source_graph_source_id: &str, + table_name: &str, ) -> Result> { if !self.ledger_exists(target_ledger_id).await? { return Ok(None); } let db = self.db(target_ledger_id).await?; - let subject = format!("{WATERMARK_SUBJECT_PREFIX}{source_graph_source_id}"); + let subject = watermark_subject(source_graph_source_id, table_name); let mut where_obj = Map::new(); where_obj.insert("@id".to_string(), JsonValue::String(subject)); where_obj.insert( @@ -225,16 +365,33 @@ impl Fluree { } } -/// Build the watermark JSON-LD node (`@id` + last snapshot id + source). Stored -/// as a string to preserve full i64 precision; `upsert` retracts-then-asserts so -/// the watermark advances cleanly in place. -fn watermark_node(source_graph_source_id: &str, to_snapshot_id: i64) -> JsonValue { +/// The per-(source, table) watermark subject IRI. Both segments are escaped +/// (`%` -> `%25`, `:` -> `%3A`) before joining with `:` so the encoding is +/// injective — distinct `(source, table)` pairs can never collide (e.g. +/// `("a:b","c")` vs `("a","b:c")`). +fn watermark_subject(source_graph_source_id: &str, table_name: &str) -> String { + fn esc(s: &str) -> String { + s.replace('%', "%25").replace(':', "%3A") + } + format!( + "{WATERMARK_SUBJECT_PREFIX}{}:{}", + esc(source_graph_source_id), + esc(table_name) + ) +} + +/// Build the watermark JSON-LD node (`@id` + last snapshot id + source + table). +/// The snapshot id is stored as a string to preserve full i64 precision; +/// `upsert` retracts-then-asserts so the watermark advances cleanly in place. +fn watermark_node( + source_graph_source_id: &str, + table_name: &str, + to_snapshot_id: i64, +) -> JsonValue { let mut node = Map::new(); node.insert( "@id".to_string(), - JsonValue::String(format!( - "{WATERMARK_SUBJECT_PREFIX}{source_graph_source_id}" - )), + JsonValue::String(watermark_subject(source_graph_source_id, table_name)), ); node.insert( WATERMARK_SNAPSHOT_PRED.to_string(), @@ -244,6 +401,10 @@ fn watermark_node(source_graph_source_id: &str, to_snapshot_id: i64) -> JsonValu WATERMARK_SOURCE_PRED.to_string(), JsonValue::String(source_graph_source_id.to_string()), ); + node.insert( + WATERMARK_TABLE_PRED.to_string(), + JsonValue::String(table_name.to_string()), + ); JsonValue::Object(node) } @@ -260,6 +421,109 @@ fn extract_first_i64(v: &JsonValue) -> Option { } } +/// The resolved latest state of one subject within a refresh window. +struct KeyState { + /// Ordering rank of the winning row: `Some` when an `order_by` column is + /// configured (numeric/timestamp/string sort key), `None` when ordering by + /// scan position only. A new row replaces the stored state when its rank is + /// `>=` the stored rank (later-in-scan breaks ties), so the latest row wins. + rank: Option<(i128, String)>, + /// The winning row's live node, or `None` if the winning row is a tombstone. + node: Option, +} + +/// Per-pass latest-by-key accumulator: one [`KeyState`] per subject IRI. The +/// BTreeMap keeps subjects in a stable order (deterministic transaction). +#[derive(Default)] +struct MaterializeAccum { + keys: BTreeMap, +} + +impl MaterializeAccum { + /// Record a classified row for `subject_iri`. `rank` is the row's ordering + /// key (from the `order_by` column) or `None` for scan-order. `node` is the + /// live node, or `None` for a tombstone. The row wins (replaces the prior + /// state) unless its rank is strictly older than the stored rank — so with an + /// ordering column the highest-ordered row wins, and without one (all ranks + /// equal `None`) the last row in scan order wins. This is a whole-row + /// REPLACE; per-subject merge across rows is the legacy additive path + /// ([`merge_live`](Self::merge_live)). + fn record( + &mut self, + subject_iri: String, + rank: Option<(i128, String)>, + node: Option, + ) { + match self.keys.get(&subject_iri) { + Some(existing) if rank < existing.rank => {} // older row: ignore + _ => { + self.keys.insert(subject_iri, KeyState { rank, node }); + } + } + } + + /// Legacy additive mode: merge a live row's node into the subject's + /// accumulated node (a subject spanning multiple source rows collects their + /// values). No ordering, no tombstones, no whole-subject retraction. + fn merge_live(&mut self, subject_iri: String, node: SubjectNode) { + match self.keys.get_mut(&subject_iri) { + Some(KeyState { + node: Some(existing), + .. + }) => existing.merge(node), + _ => { + self.keys.insert( + subject_iri, + KeyState { + rank: None, + node: Some(node), + }, + ); + } + } + } + + /// Resolve into `(live nodes to assert, subject IRIs whose latest row is a + /// tombstone)`. + fn finalize(self) -> (BTreeMap, BTreeSet) { + let mut live = BTreeMap::new(); + let mut deletions = BTreeSet::new(); + for (iri, state) in self.keys { + match state.node { + Some(node) => { + live.insert(iri, node); + } + None => { + deletions.insert(iri); + } + } + } + (live, deletions) + } +} + +/// Build the whole-subject retraction transaction for a set of subject IRIs: +/// bind every `(?p, ?o)` for each listed `?s` and delete it. An IRI that was +/// never materialized binds zero rows -> zero flakes -> harmless no-op. +/// +/// Each subject is bound as a typed `@id` value (`{"@type":"@id","@value":...}`) +/// — NOT a bare string. A bare string parses to a string *literal*, which never +/// joins against a real subject Sid, so the wildcard delete would silently match +/// zero rows and retract nothing. The doc carries no `@context`, and the +/// materialized IRIs are already fully expanded, so the bound Sid matches the +/// `@id` the upsert asserted. (Shape proven in `it_transact_update.rs`.) +fn build_retract_doc(iris: &BTreeSet) -> JsonValue { + let rows: Vec = iris + .iter() + .map(|iri| JsonValue::Array(vec![json!({ "@type": "@id", "@value": iri })])) + .collect(); + json!({ + "values": ["?s", rows], + "where": { "@id": "?s", "?p": "?o" }, + "delete": { "@id": "?s", "?p": "?o" } + }) +} + /// Accumulates all triples for a single subject IRI before emitting one JSON-LD /// node. Predicates collect multiple object values (combined into an array on /// output) and `@type` classes are de-duplicated. @@ -292,6 +556,19 @@ impl SubjectNode { } } + /// Merge another node's types and objects into this one (legacy additive + /// mode: a subject that spans multiple source rows accumulates their values). + fn merge(&mut self, other: SubjectNode) { + for t in other.types { + self.add_type(&t); + } + for (pred, vals) in other.predicates { + for v in vals { + self.add_object(&pred, v); + } + } + } + fn into_json(self) -> JsonValue { let mut node = Map::new(); node.insert("@id".to_string(), JsonValue::String(self.id)); @@ -313,35 +590,19 @@ impl SubjectNode { } } -/// Expand one source row through a triples map and merge its triples into the -/// per-subject accumulator. Reuses the exact `term.rs` materializers so the -/// subject IRIs match the query path byte-for-byte. -fn materialize_row_into( +/// Build a live `SubjectNode` from one source row (subject classes + the +/// constant-predicate objects). RefObjectMap joins are resolved at query time, +/// not during materialization. +fn build_live_node( tm: &TriplesMap, batch: &ColumnBatch, row: usize, - subjects: &mut BTreeMap, -) -> Result<()> { - let subject_term = materialize_subject_from_batch(&tm.subject_map, batch, row) - .map_err(|e| ApiError::Internal(format!("R2RML subject materialization failed: {e}")))?; - let subject_iri = match subject_term { - Some(RdfTerm::Iri(iri)) => iri, - // Null subject column (skip), blank-node subjects (no stable identity to - // upsert), or a literal (term.rs already rejects this) -> skip the row. - Some(RdfTerm::BlankNode(_) | RdfTerm::Literal { .. }) | None => return Ok(()), - }; - - let node = subjects - .entry(subject_iri.clone()) - .or_insert_with(|| SubjectNode::new(subject_iri)); - - // rr:class -> @type + id: String, +) -> Result { + let mut node = SubjectNode::new(id); for class in &tm.subject_map.classes { node.add_type(class); } - - // predicate-object maps (constant predicates only; RefObjectMap joins are - // resolved at query time, not yet during materialization). for pom in &tm.predicate_object_maps { let Some(predicate) = pom.predicate_map.as_constant() else { continue; @@ -352,7 +613,55 @@ fn materialize_row_into( node.add_object(predicate, rdf_term_to_jsonld(term)); } } + Ok(node) +} + +/// Expand one source row through a triples map into the accumulator. The subject +/// IRI is built by the exact `term.rs` materializer (so live and tombstone rows +/// for the same key produce the same IRI). +/// +/// When `latest_by_key` (a `delete` convention and/or `order_by` is configured), +/// the row is classified live vs tombstone, ranked by `order_by`, and recorded +/// as a whole-row REPLACE — the latest row per key wins. Otherwise (legacy +/// additive mode) the live row's triples are MERGED into the subject's node, so +/// a subject spanning multiple rows accumulates their values. +fn materialize_row_into( + tm: &TriplesMap, + batch: &ColumnBatch, + row: usize, + convention: Option<&DeleteConvention>, + order_by: Option<&str>, + latest_by_key: bool, + accum: &mut MaterializeAccum, +) -> Result<()> { + let subject_term = materialize_subject_from_batch(&tm.subject_map, batch, row) + .map_err(|e| ApiError::Internal(format!("R2RML subject materialization failed: {e}")))?; + let subject_iri = match subject_term { + Some(RdfTerm::Iri(iri)) => iri, + // Null subject column (skip), blank-node subjects (no stable identity to + // upsert), or a literal (term.rs already rejects this) -> skip the row. + Some(RdfTerm::BlankNode(_) | RdfTerm::Literal { .. }) | None => return Ok(()), + }; + + if !latest_by_key { + // Legacy additive: merge this live row into the subject's node. + let node = build_live_node(tm, batch, row, subject_iri.clone())?; + accum.merge_live(subject_iri, node); + return Ok(()); + } + // Latest-by-key: classify, rank, and record as a whole-row replace. + let rank = order_by.and_then(|col| column_sort_key(batch, col, row)); + let is_tombstone = convention.is_some_and(|conv| { + let value = column_string(batch, &conv.column, row); + conv.is_tombstone(value.as_deref()) + }); + let node = if is_tombstone { + None + } else { + Some(build_live_node(tm, batch, row, subject_iri.clone())?) + }; + accum.record(subject_iri, rank, node); Ok(()) } @@ -383,3 +692,129 @@ fn rdf_term_to_jsonld(term: RdfTerm) -> JsonValue { }, } } + +#[cfg(test)] +mod tests { + use super::*; + + fn live(iri: &str) -> Option { + Some(SubjectNode::new(iri.to_string())) + } + fn ts(n: i128) -> Option<(i128, String)> { + Some((n, String::new())) + } + + #[test] + fn finalize_live_only_upserts() { + let mut a = MaterializeAccum::default(); + a.record("urn:a".into(), None, live("urn:a")); + let (live, del) = a.finalize(); + assert!(live.contains_key("urn:a")); + assert!(del.is_empty()); + } + + #[test] + fn finalize_tombstone_only_retracts() { + let mut a = MaterializeAccum::default(); + a.record("urn:a".into(), None, None); + let (live, del) = a.finalize(); + assert!(live.is_empty()); + assert!(del.contains("urn:a")); + } + + #[test] + fn scan_order_last_wins_live_then_tombstone() { + // No ordering column: last row in scan order wins -> tombstone. + let mut a = MaterializeAccum::default(); + a.record("urn:a".into(), None, live("urn:a")); + a.record("urn:a".into(), None, None); + let (live, del) = a.finalize(); + assert!(live.is_empty()); + assert!(del.contains("urn:a")); + } + + #[test] + fn scan_order_last_wins_tombstone_then_live() { + let mut a = MaterializeAccum::default(); + a.record("urn:a".into(), None, None); + a.record("urn:a".into(), None, live("urn:a")); + let (live, del) = a.finalize(); + assert!(live.contains_key("urn:a")); + assert!(!del.contains("urn:a")); + } + + #[test] + fn order_by_latest_wins_regardless_of_arrival() { + // A higher-ranked tombstone arriving FIRST still wins over a lower-ranked + // live row arriving later (ordering, not scan order, decides). + let mut a = MaterializeAccum::default(); + a.record("urn:a".into(), ts(200), None); // newer tombstone + a.record("urn:a".into(), ts(100), live("urn:a")); // older live + let (live, del) = a.finalize(); + assert!(live.is_empty()); + assert!(del.contains("urn:a")); + } + + #[test] + fn order_by_newer_live_wins_over_older_tombstone() { + let mut a = MaterializeAccum::default(); + a.record("urn:a".into(), ts(100), None); // older tombstone + a.record("urn:a".into(), ts(200), live("urn:a")); // newer live + let (live, del) = a.finalize(); + assert!(live.contains_key("urn:a")); + assert!(!del.contains("urn:a")); + } + + #[test] + fn build_retract_doc_shape() { + let mut set = BTreeSet::new(); + set.insert("urn:a".to_string()); + set.insert("urn:b".to_string()); + let doc = build_retract_doc(&set); + // Subjects MUST be typed @id values, not bare strings (a bare string + // parses to a literal that never joins a real subject Sid). BTreeSet + // => sorted order. + assert_eq!(doc["values"][0], json!("?s")); + assert_eq!( + doc["values"][1], + json!([ + [{ "@type": "@id", "@value": "urn:a" }], + [{ "@type": "@id", "@value": "urn:b" }] + ]) + ); + assert_eq!(doc["where"], json!({ "@id": "?s", "?p": "?o" })); + assert_eq!(doc["delete"], json!({ "@id": "?s", "?p": "?o" })); + } + + #[test] + fn watermark_node_is_per_table_and_string_encoded() { + let node = watermark_node("people:main", "demo.actors", 5_648_190_075_564_901_028); + // Source's ':' is escaped (%3A) so (source, table) encoding is injective. + assert_eq!( + node["@id"], + json!("urn:fluree:materialize-state:people%3Amain:demo.actors") + ); + // String-encoded to preserve full i64 precision. + assert_eq!(node[WATERMARK_SNAPSHOT_PRED], json!("5648190075564901028")); + assert_eq!(node[WATERMARK_SOURCE_PRED], json!("people:main")); + assert_eq!(node[WATERMARK_TABLE_PRED], json!("demo.actors")); + } + + #[test] + fn watermark_subject_is_injective() { + // Distinct (source, table) pairs that would collide under a naive ':' + // join must produce distinct subjects. + assert_ne!(watermark_subject("a:b", "c"), watermark_subject("a", "b:c")); + } + + #[test] + fn extract_first_i64_tolerant_of_nesting() { + assert_eq!(extract_first_i64(&json!(["42"])), Some(42)); + assert_eq!(extract_first_i64(&json!([["42"]])), Some(42)); + assert_eq!(extract_first_i64(&json!([])), None); + assert_eq!( + extract_first_i64(&json!(["5648190075564901028"])), + Some(5_648_190_075_564_901_028) + ); + } +} diff --git a/fluree-db-api/src/lib.rs b/fluree-db-api/src/lib.rs index 7a48cca03c..7510277cc3 100644 --- a/fluree-db-api/src/lib.rs +++ b/fluree-db-api/src/lib.rs @@ -195,6 +195,9 @@ pub use graph_source::{ R2rmlCreateConfig, R2rmlCreateResult, R2rmlMappingInput, RestCatalogMode, }; +#[cfg(feature = "iceberg")] +pub use fluree_db_iceberg::DeleteConvention; + pub use bm25_worker::{ Bm25MaintenanceWorker, Bm25WorkerConfig, Bm25WorkerHandle, Bm25WorkerState, Bm25WorkerStats, }; diff --git a/fluree-db-api/src/materialize_worker.rs b/fluree-db-api/src/materialize_worker.rs index 17970a135e..556aaf530e 100644 --- a/fluree-db-api/src/materialize_worker.rs +++ b/fluree-db-api/src/materialize_worker.rs @@ -287,6 +287,7 @@ impl MaterializeTrackingWorker { incremental = result.incremental, rows_read = result.rows_read, subjects_upserted = result.subjects_upserted, + subjects_retracted = result.subjects_retracted, "materialize tracking synced" ); } diff --git a/fluree-db-api/tests/grp_graphsource.rs b/fluree-db-api/tests/grp_graphsource.rs index 1eb44db7f3..cdab6352fe 100644 --- a/fluree-db-api/tests/grp_graphsource.rs +++ b/fluree-db-api/tests/grp_graphsource.rs @@ -15,6 +15,8 @@ mod it_default_context; mod it_graph_commit; #[path = "it_graph_source_bm25.rs"] mod it_graph_source_bm25; +#[path = "it_materialize_retract.rs"] +mod it_materialize_retract; #[path = "it_named_graph_isolation.rs"] mod it_named_graph_isolation; #[path = "it_named_graphs.rs"] diff --git a/fluree-db-api/tests/it_materialize_retract.rs b/fluree-db-api/tests/it_materialize_retract.rs new file mode 100644 index 0000000000..7549a70963 --- /dev/null +++ b/fluree-db-api/tests/it_materialize_retract.rs @@ -0,0 +1,97 @@ +//! Regression tests for the whole-subject retraction shape used by R2RML +//! materialization deletion (`build_retract_doc`). +//! +//! The wildcard retraction MUST bind `?s` as a typed `@id` value. A bare-string +//! binding parses to a string *literal* that never joins a real subject Sid, so +//! the delete silently matches zero rows and retracts nothing. These tests run +//! the shape end-to-end through `Fluree::update` against a memory ledger and +//! assert the triples are actually gone — exactly what was missing when the +//! bare-string bug shipped. + +use crate::support; +use fluree_db_api::FlureeBuilder; +use serde_json::json; + +const NAME: &str = "https://www.w3.org/ns/activitystreams#name"; + +#[tokio::test] +async fn materialize_retract_shape_actually_retracts() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger0 = support::genesis_ledger(&fluree, "mat/retract:main"); + + let seed = json!({ + "@graph": [ + { + "@id": "https://example.org/Actor%2F1", + "https://www.w3.org/ns/activitystreams#name": "Alice", + "https://www.w3.org/ns/activitystreams#summary": "first" + }, + { + "@id": "https://example.org/Actor%2F2", + "https://www.w3.org/ns/activitystreams#name": "Bob" + } + ] + }); + let ledger = fluree.insert(ledger0, &seed).await.unwrap().ledger; + + // Whole-subject retraction of Actor/1 using the CORRECT typed-@id values + // shape (mirrors build_retract_doc). + let retract = json!({ + "values": ["?s", [[{ "@type": "@id", "@value": "https://example.org/Actor%2F1" }]]], + "where": { "@id": "?s", "?p": "?o" }, + "delete": { "@id": "?s", "?p": "?o" } + }); + let ledger = fluree.update(ledger, &retract).await.unwrap().ledger; + + // Actor/1 is fully gone (all predicates); Actor/2 remains. + let q = json!({ + "select": ["?s"], + "where": { "@id": "?s", "https://www.w3.org/ns/activitystreams#name": "?n" } + }); + let out = support::query_jsonld_formatted(&fluree, &ledger, &q) + .await + .unwrap() + .to_string(); + assert!( + !out.contains("Actor%2F1"), + "Actor/1 should be retracted, got: {out}" + ); + assert!( + out.contains("Actor%2F2"), + "Actor/2 should remain, got: {out}" + ); +} + +#[tokio::test] +async fn materialize_retract_bare_string_is_noop() { + // Control: the bare-string binding (the shipped bug) retracts nothing. + let fluree = FlureeBuilder::memory().build_memory(); + let ledger0 = support::genesis_ledger(&fluree, "mat/retract-bare:main"); + let seed = json!({ + "@graph": [ + { "@id": "https://example.org/Actor%2F1", "https://www.w3.org/ns/activitystreams#name": "Alice" } + ] + }); + let ledger = fluree.insert(ledger0, &seed).await.unwrap().ledger; + + let retract = json!({ + "values": ["?s", [["https://example.org/Actor%2F1"]]], + "where": { "@id": "?s", "?p": "?o" }, + "delete": { "@id": "?s", "?p": "?o" } + }); + let ledger = fluree.update(ledger, &retract).await.unwrap().ledger; + + let q = json!({ + "select": ["?s"], + "where": { "@id": "?s", "https://www.w3.org/ns/activitystreams#name": "?n" } + }); + let out = support::query_jsonld_formatted(&fluree, &ledger, &q) + .await + .unwrap() + .to_string(); + assert!( + out.contains("Actor%2F1"), + "bare-string retraction must be a no-op (subject still present), got: {out}" + ); + let _ = NAME; // documents the predicate under test +} diff --git a/fluree-db-iceberg/src/config.rs b/fluree-db-iceberg/src/config.rs index b7e04da0f3..1e8313278c 100644 --- a/fluree-db-iceberg/src/config.rs +++ b/fluree-db-iceberg/src/config.rs @@ -44,6 +44,75 @@ pub struct IcebergGsConfig { /// R2RML mapping source (format-agnostic, used in Phase 3) #[serde(default)] pub mapping: Option, + /// Optional tombstone/delete convention. When set, materialization + /// classifies each source row as live or a delete (tombstone) and retracts + /// tombstoned subjects from the target ledger. Absent => additive (no + /// retraction), which is the legacy behavior. + #[serde(default)] + pub delete: Option, + /// Optional ordering column for latest-by-key materialization. When set, the + /// rows of each subject within a refresh window are ordered by this column + /// (numeric/timestamp columns compare by value; others lexicographically) and + /// the **latest** row defines the subject — a whole-subject replace that + /// clears fields dropped in the newer revision, matching a + /// `ROW_NUMBER() … ORDER BY DESC` latest-by-key view. Absent => the + /// last row in scan order wins and live revisions are merged per predicate + /// (legacy behavior; fields cleared in a later revision are NOT removed). + #[serde(default)] + pub order_by: Option, +} + +/// Declares how a delete is encoded in the source table's append log so the +/// materializer can recognize "tombstone" rows and retract those subjects. +/// +/// Append-only CDC sinks model a delete as an ordinary appended row carrying a +/// marker. A row is a tombstone when the value of `column` is one of +/// `deleted_values`. A `null` entry in `deleted_values` matches a NULL `column` +/// value — the Debezium null-payload convention (used when the table has no +/// explicit op column). Examples: +/// - `{ column: "_op", deleted_values: ["d", "delete"] }` — value-match op column. +/// - `{ column: "type", deleted_values: [null] }` — null-payload delete. +/// - `{ column: "_op", deleted_values: ["d", null] }` — either. +/// +/// The subject IRI of a tombstone row is derived by the SAME R2RML subject +/// materializer as a live row, so the retracted IRI matches what was asserted. +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct DeleteConvention { + /// Source column inspected to classify a row as a tombstone. + pub column: String, + /// Column values that mark a row as a delete (tombstone). A `null` element + /// matches a NULL `column` value (null-payload delete). Must be non-empty. + #[serde(default)] + pub deleted_values: Vec>, +} + +impl DeleteConvention { + /// Validate the convention is usable: a column is named and at least one + /// delete value (possibly `null`) is declared. + pub fn validate(&self) -> Result<()> { + if self.column.trim().is_empty() { + return Err(IcebergError::Config( + "delete.column is required".to_string(), + )); + } + if self.deleted_values.is_empty() { + return Err(IcebergError::Config( + "delete.deleted_values must list at least one value (use null for a \ + null-payload delete)" + .to_string(), + )); + } + Ok(()) + } + + /// Classify one row by its marker-column value (already converted to its + /// string form, or `None` when the column is null for that row): `true` if + /// that value — including `None` for a null — is in `deleted_values`. + pub fn is_tombstone(&self, column_value: Option<&str>) -> bool { + self.deleted_values + .iter() + .any(|d| d.as_deref() == column_value) + } } impl IcebergGsConfig { @@ -134,6 +203,10 @@ impl IcebergGsConfig { } } + if let Some(delete) = &self.delete { + delete.validate()?; + } + Ok(()) } } @@ -510,6 +583,8 @@ mod tests { table: TableConfig::Identifier("ns.table".to_string()), io: IoConfig::default(), mapping: None, + delete: None, + order_by: None, }; let result = config.validate(); assert!(result.is_err()); @@ -523,6 +598,8 @@ mod tests { table: TableConfig::Identifier("invalid".to_string()), io: IoConfig::default(), mapping: None, + delete: None, + order_by: None, }; assert!(config.validate().is_err()); } @@ -539,6 +616,8 @@ mod tests { ..Default::default() }, mapping: None, + delete: None, + order_by: None, }; let result = config.validate(); assert!(result.is_err()); @@ -555,6 +634,8 @@ mod tests { ..Default::default() }, mapping: None, + delete: None, + order_by: None, }; let result = config.validate(); assert!(result.is_err()); @@ -571,6 +652,8 @@ mod tests { ..Default::default() }, mapping: None, + delete: None, + order_by: None, }; let result = config.validate(); assert!(result.is_err()); @@ -580,6 +663,82 @@ mod tests { .contains("Vended credentials")); } + // ── Delete convention ── + + fn dv(values: &[Option<&str>]) -> Vec> { + values + .iter() + .map(|v| v.map(std::string::ToString::to_string)) + .collect() + } + + #[test] + fn test_delete_convention_validate() { + // value-match + assert!(DeleteConvention { + column: "_op".to_string(), + deleted_values: dv(&[Some("d"), Some("delete")]), + } + .validate() + .is_ok()); + // null-payload (null entry) + assert!(DeleteConvention { + column: "type".to_string(), + deleted_values: dv(&[None]), + } + .validate() + .is_ok()); + // empty column rejected + assert!(DeleteConvention { + column: String::new(), + deleted_values: dv(&[Some("d")]), + } + .validate() + .is_err()); + // no delete values rejected + assert!(DeleteConvention { + column: "_op".to_string(), + deleted_values: vec![], + } + .validate() + .is_err()); + } + + #[test] + fn test_delete_convention_is_tombstone() { + let value = DeleteConvention { + column: "_op".to_string(), + deleted_values: dv(&[Some("d")]), + }; + assert!(value.is_tombstone(Some("d"))); + assert!(!value.is_tombstone(Some("c"))); + assert!(!value.is_tombstone(None)); // null is NOT a delete unless listed + + let null = DeleteConvention { + column: "type".to_string(), + deleted_values: dv(&[None]), + }; + assert!(null.is_tombstone(None)); // null payload => tombstone + assert!(!null.is_tombstone(Some("Profile"))); + + // both: a value OR null marks a delete + let both = DeleteConvention { + column: "_op".to_string(), + deleted_values: dv(&[Some("d"), None]), + }; + assert!(both.is_tombstone(Some("d"))); + assert!(both.is_tombstone(None)); + assert!(!both.is_tombstone(Some("u"))); + } + + #[test] + fn test_delete_convention_serde_default_absent() { + // A config without `delete` deserializes with delete == None (backward compat). + let json = r#"{"catalog":{"type":"direct","table_location":"s3://b/w/ns/t"},"table":""}"#; + let cfg = IcebergGsConfig::from_json(json).unwrap(); + assert!(cfg.delete.is_none()); + } + // ── Roundtrip serialization ── #[test] @@ -594,6 +753,8 @@ mod tests { table: TableConfig::Identifier("ns.table".to_string()), io: IoConfig::default(), mapping: None, + delete: None, + order_by: None, }; let json = original.to_json().unwrap(); @@ -612,6 +773,8 @@ mod tests { ..Default::default() }, mapping: None, + delete: None, + order_by: None, }; let json = original.to_json().unwrap(); diff --git a/fluree-db-iceberg/src/lib.rs b/fluree-db-iceberg/src/lib.rs index 0da48075a6..d0d35669e6 100644 --- a/fluree-db-iceberg/src/lib.rs +++ b/fluree-db-iceberg/src/lib.rs @@ -63,7 +63,9 @@ pub mod metadata; pub mod scan; // Re-export commonly used types -pub use config::{CatalogConfig, IcebergGsConfig, IoConfig, MappingSource, TableConfig}; +pub use config::{ + CatalogConfig, DeleteConvention, IcebergGsConfig, IoConfig, MappingSource, TableConfig, +}; pub use config_value::ConfigValue; pub use error::{IcebergError, Result}; diff --git a/fluree-db-r2rml/src/materialize/mod.rs b/fluree-db-r2rml/src/materialize/mod.rs index b051ad49fb..9181eafc69 100644 --- a/fluree-db-r2rml/src/materialize/mod.rs +++ b/fluree-db-r2rml/src/materialize/mod.rs @@ -23,6 +23,7 @@ pub use term::reverse_subject_template; // ColumnBatch-based API (for production efficiency) pub use term::{ + batch_has_column, column_is_orderable, column_sort_key, column_string, expand_template_from_batch, get_join_key_from_batch, materialize_object_from_batch, materialize_subject_from_batch, }; diff --git a/fluree-db-r2rml/src/materialize/term.rs b/fluree-db-r2rml/src/materialize/term.rs index decbdefe22..e5d9e280a7 100644 --- a/fluree-db-r2rml/src/materialize/term.rs +++ b/fluree-db-r2rml/src/materialize/term.rs @@ -502,6 +502,78 @@ fn column_value_as_string( column_to_string(col, row_idx) } +/// Read a column's value at `row_idx` as a string, using the exact same +/// conversion the subject/object materializers use. Intended for reading +/// auxiliary columns (e.g. a delete-marker column) so comparisons match the +/// source encoding (booleans as `true`/`false`, timestamps/decimals formatted +/// identically). Returns `None` when the column is absent OR the value is null +/// — use [`batch_has_column`] to distinguish the two. +pub fn column_string(batch: &ColumnBatch, column_name: &str, row_idx: usize) -> Option { + column_value_as_string(batch, column_name, row_idx) +} + +/// Whether `column_name` exists in `batch` (to validate a configured +/// marker column independently of whether a given row's value is null). +pub fn batch_has_column(batch: &ColumnBatch, column_name: &str) -> bool { + batch.column_by_name(column_name).is_some() +} + +/// A total-order sort key for an ordering column value at `row_idx`, used for +/// latest-by-key materialization. Numeric and temporal columns (int, date, +/// timestamp) compare by their integer value; all other columns compare +/// lexicographically by their string form. The returned `(i128, String)` +/// compares as a tuple, so within one column (one type) the comparison is the +/// natural one. `None` when the column is absent or null for that row (treated +/// by callers as the smallest / "no ordering info"). +pub fn column_sort_key( + batch: &ColumnBatch, + column_name: &str, + row_idx: usize, +) -> Option<(i128, String)> { + let col = batch.column_by_name(column_name)?; + match col { + Column::Int32(v) => v + .get(row_idx) + .and_then(|v| *v) + .map(|n| (i128::from(n), String::new())), + Column::Int64(v) => v + .get(row_idx) + .and_then(|v| *v) + .map(|n| (i128::from(n), String::new())), + Column::Date(v) => v + .get(row_idx) + .and_then(|v| *v) + .map(|n| (i128::from(n), String::new())), + Column::Timestamp(v) | Column::TimestampTz(v) => v + .get(row_idx) + .and_then(|v| *v) + .map(|n| (i128::from(n), String::new())), + // Floats and other types: fall back to string ordering (numeric part 0). + // Callers that require value-correct ordering must gate on + // [`column_is_orderable`] first (the materializer rejects non-orderable + // `order_by` columns), so this arm is only a best-effort fallback. + _ => column_to_string(col, row_idx).map(|s| (0i128, s)), + } +} + +/// Whether `column_name` is a type [`column_sort_key`] orders by value (integer, +/// date, or timestamp). Float/decimal/string/boolean are NOT value-orderable by +/// the sort key (they fall back to a lexicographic string, which mis-orders +/// numbers like `10 < 9`), so a latest-by-key `order_by` column must be one of +/// these types. Returns `false` if the column is absent. +pub fn column_is_orderable(batch: &ColumnBatch, column_name: &str) -> bool { + matches!( + batch.column_by_name(column_name), + Some( + Column::Int32(_) + | Column::Int64(_) + | Column::Date(_) + | Column::Timestamp(_) + | Column::TimestampTz(_) + ) + ) +} + /// Convert a Column value at a row index to a String fn column_to_string(col: &Column, row_idx: usize) -> Option { match col { diff --git a/fluree-db-server/src/routes/iceberg.rs b/fluree-db-server/src/routes/iceberg.rs index e7176c47fe..494ca56997 100644 --- a/fluree-db-server/src/routes/iceberg.rs +++ b/fluree-db-server/src/routes/iceberg.rs @@ -59,6 +59,16 @@ pub struct IcebergMapRequest { /// Use path-style S3 URLs #[serde(default)] pub s3_path_style: bool, + /// Tombstone/delete convention: source column inspected to classify a row + /// as a delete during materialization. Omit to disable retraction. + pub delete_column: Option, + /// Column values that mark a row as a delete. A `null` element matches a NULL + /// `delete_column` value (null-payload tombstone), e.g. `["d", "delete"]`, + /// `[null]`, or `["d", null]`. Required when `delete_column` is set. + #[serde(default)] + pub delete_values: Vec>, + /// Ordering column for latest-by-key materialization (e.g. `event_timestamp`). + pub order_by: Option, } fn default_mode() -> String { @@ -209,6 +219,7 @@ pub struct IcebergMaterializeResponse { pub committed: bool, pub rows_read: usize, pub subjects_upserted: usize, + pub subjects_retracted: usize, } /// Materialize an R2RML / Iceberg graph source into a native ledger. @@ -271,6 +282,7 @@ async fn iceberg_materialize_local( committed: result.committed, rows_read: result.rows_read, subjects_upserted: result.subjects_upserted, + subjects_retracted: result.subjects_retracted, }; tracing::info!( @@ -282,6 +294,7 @@ async fn iceberg_materialize_local( committed = response.committed, rows_read = response.rows_read, subjects_upserted = response.subjects_upserted, + subjects_retracted = response.subjects_retracted, "iceberg materialize complete" ); Ok((StatusCode::OK, Json(response))) @@ -376,6 +389,7 @@ async fn iceberg_track_local(state: Arc, request: Request) -> Result Result Date: Thu, 2 Jul 2026 11:14:37 +0200 Subject: [PATCH 5/5] feat(iceberg): refreshable Google metadata-server catalog auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A static bearer for a Google Iceberg REST catalog (BigLake) is a short-lived OAuth access token: it expires after ~1h and `BearerTokenAuth` cannot renew it, so a long-running materialization tracking worker starts returning 401s. Add `AuthConfig::GoogleMetadata` + `GoogleMetadataAuth`, which mints and auto-refreshes tokens from the GCE/GKE instance metadata server (Workload Identity), mirroring the existing `OAuth2ClientCredentials` cache+refresh — the jittered-expiry `CachedToken` is now shared via `auth::token`. Wire it through `{Iceberg,R2rml}CreateConfig::with_auth_google_metadata` and the `/iceberg/map` `auth_google_metadata` field. The GCS reader's storage HMAC keys are unaffected (static, non-expiring). The metadata server is only reachable on GCE/GKE; local runs keep using a static `auth_bearer`. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/api/endpoints.md | 6 +- docs/graph-sources/iceberg.md | 4 +- fluree-db-api/src/graph_source/config.rs | 25 ++ fluree-db-iceberg/src/auth/google_metadata.rs | 243 ++++++++++++++++++ fluree-db-iceberg/src/auth/mod.rs | 63 +++++ fluree-db-iceberg/src/auth/oauth2.rs | 29 +-- fluree-db-iceberg/src/auth/token.rs | 35 +++ fluree-db-server/src/routes/iceberg.rs | 12 + 8 files changed, 387 insertions(+), 30 deletions(-) create mode 100644 fluree-db-iceberg/src/auth/google_metadata.rs create mode 100644 fluree-db-iceberg/src/auth/token.rs diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index 42d1554bc0..35cb09917b 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -2542,8 +2542,10 @@ POST http://localhost:8090/v1/fluree/iceberg/map | `r2rml` | string | Inline R2RML mapping (Turtle/JSON-LD). Omit to auto-generate a direct mapping. | | `r2rml_type` | string | Media type of `r2rml` (`text/turtle`, `application/ld+json`) | | `branch` | string | Branch name (default: `main`) | -| `auth_bearer` | string | Bearer token for catalog auth | -| `oauth2_*` | string | OAuth2 client-credentials flow for the catalog | +| `auth_bearer` | string | Static bearer token for catalog auth (does not refresh — a Google OAuth token will expire after ~1h) | +| `oauth2_*` | string | OAuth2 client-credentials flow for the catalog (refreshes) | +| `auth_google_metadata` | bool | Use the GCE/GKE metadata server (Workload Identity) for catalog auth, minting + auto-refreshing tokens — for Google Iceberg REST catalogs (BigLake). Overrides `auth_bearer`. Only works when running on GCP. | +| `auth_google_scopes` | string | Optional OAuth scopes for `auth_google_metadata` (default `cloud-platform`) | | `warehouse` | string | Warehouse identifier | | `no_vended_credentials` | bool | Disable vended credentials | | `s3_region`, `s3_endpoint`, `s3_path_style` | | S3 overrides for `direct` mode | diff --git a/docs/graph-sources/iceberg.md b/docs/graph-sources/iceberg.md index eb626f4d6d..5aa68187d0 100644 --- a/docs/graph-sources/iceberg.md +++ b/docs/graph-sources/iceberg.md @@ -237,7 +237,9 @@ export AWS_SECRET_ACCESS_KEY= A signing region is required and must match the bucket location — SigV4 scopes the signature to a region, and GCS interop rejects a mismatched or unsigned region. Set it via `s3_region` in the config (recommended, and what the examples use) or via the ambient `AWS_REGION` in the server environment. `s3_endpoint` must be the interop host, and `s3_path_style` must be `true`. HMAC keys do not expire; in `rest` mode, credentials vended by the catalog for a GCS-backed table are used instead (and refreshed by the SDK). -GCS-backed Iceberg tables are typically read via `direct` mode — point `table_location` at the table root. GCS-native conventions are handled automatically: `gs://` paths in metadata/manifests, a Hadoop-style integer `version-hint.text` (resolved to `vN.metadata.json`), and Snappy-compressed Parquet. As with any direct-mode table, the Iceberg layout (the `metadata/` directory and a current `version-hint.text`) must already exist in the bucket. +GCS-backed Iceberg tables are typically read via `direct` mode — point `table_location` at the table root. GCS-native conventions are handled automatically: `gs://` paths in metadata/manifests, a Hadoop-style integer `version-hint.text` (resolved to `vN.metadata.json`), and Snappy/GZIP-compressed Parquet. As with any direct-mode table, the Iceberg layout (the `metadata/` directory and a current `version-hint.text`) must already exist in the bucket. + +**BigLake REST catalog (catalog auth, distinct from the storage HMAC above).** For tables discovered through Google's BigLake Iceberg REST catalog, the catalog `loadTable` call authenticates with a **Google OAuth token** — separate from the HMAC keys that read the `gs://` data files. A **static `auth_bearer`** (e.g. `gcloud auth print-access-token`) works for a one-shot map/query but **expires after ~1h and cannot renew**, so a long-running tracking worker starts returning 401s. For a workload running as a GCP service account (GKE **Workload Identity**), set **`auth_google_metadata: true`** instead: it mints and auto-refreshes tokens from the instance metadata server, so tracked jobs keep authenticating. (The metadata server is only reachable on GCE/GKE; locally, use a static `auth_bearer`.) The storage HMAC keys are unaffected — they don't expire. ## RDF Mapping (R2RML) diff --git a/fluree-db-api/src/graph_source/config.rs b/fluree-db-api/src/graph_source/config.rs index 3886e925f7..b3e189c433 100644 --- a/fluree-db-api/src/graph_source/config.rs +++ b/fluree-db-api/src/graph_source/config.rs @@ -512,6 +512,24 @@ impl IcebergCreateConfig { self } + /// Set Google metadata-server authentication (REST mode only). + /// + /// Mints and refreshes short-lived Google OAuth tokens from the GCE/GKE + /// metadata server (Workload Identity) — for Google Iceberg REST catalogs + /// (BigLake), where a static bearer expires after ~1h. `scopes` is optional + /// (defaults to cloud-platform). + pub fn with_auth_google_metadata(mut self, scopes: Option) -> Self { + if let CatalogMode::Rest(ref mut rest) = self.catalog_mode { + rest.auth = fluree_db_iceberg::auth::AuthConfig::GoogleMetadata { + scopes, + metadata_url: None, + }; + } else { + tracing::warn!("with_auth_google_metadata has no effect in Direct catalog mode"); + } + self + } + /// Set OAuth2 client credentials authentication (REST mode only). pub fn with_auth_oauth2( mut self, @@ -874,6 +892,13 @@ impl R2rmlCreateConfig { self } + /// Set Google metadata-server authentication (GKE Workload Identity), with + /// automatic token refresh. See [`IcebergCreateConfig::with_auth_google_metadata`]. + pub fn with_auth_google_metadata(mut self, scopes: Option) -> Self { + self.iceberg = self.iceberg.with_auth_google_metadata(scopes); + self + } + /// Set OAuth2 client credentials authentication. pub fn with_auth_oauth2( mut self, diff --git a/fluree-db-iceberg/src/auth/google_metadata.rs b/fluree-db-iceberg/src/auth/google_metadata.rs new file mode 100644 index 0000000000..f0d913d95a --- /dev/null +++ b/fluree-db-iceberg/src/auth/google_metadata.rs @@ -0,0 +1,243 @@ +//! Google metadata-server token authentication. +//! +//! Mints and refreshes short-lived Google OAuth access tokens from the GCE/GKE +//! instance metadata server — the credential model for a workload running as a +//! GCP service account (GKE Workload Identity). Unlike a static bearer token +//! (which Google issues with a ~1h lifetime and cannot be renewed once set), this +//! provider refreshes internally, so a long-running catalog client — notably the +//! materialization tracking worker polling a Google Iceberg REST catalog +//! (BigLake) — keeps authenticating past the first hour. +//! +//! The metadata server is only reachable from within GCE/GKE; for local runs use +//! a static `bearer` token instead. + +use crate::auth::token::CachedToken; +use crate::auth::{CatalogAuth, SendCatalogAuth}; +use crate::error::{IcebergError, Result}; +use async_trait::async_trait; +use chrono::{Duration, Utc}; +use std::sync::Arc; +use tokio::sync::RwLock; + +/// GCE/GKE metadata-server token endpoint for the attached (default) service +/// account. +const DEFAULT_METADATA_TOKEN_URL: &str = + "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; + +/// Default OAuth scope for Google Cloud APIs (covers the BigLake REST catalog). +const DEFAULT_SCOPES: &str = "https://www.googleapis.com/auth/cloud-platform"; + +/// Configuration for [`GoogleMetadataAuth`]. +#[derive(Debug, Clone)] +pub struct GoogleMetadataConfig { + /// Token endpoint. Defaults to the metadata server; overridable for testing. + pub metadata_url: String, + /// OAuth scopes (comma-separated). Defaults to cloud-platform. + pub scopes: String, +} + +impl Default for GoogleMetadataConfig { + fn default() -> Self { + Self { + metadata_url: DEFAULT_METADATA_TOKEN_URL.to_string(), + scopes: DEFAULT_SCOPES.to_string(), + } + } +} + +/// Google metadata-server OAuth authentication. +/// +/// Acquires tokens from the instance metadata server and refreshes them +/// automatically before expiration (see [`CachedToken`]). +pub struct GoogleMetadataAuth { + config: GoogleMetadataConfig, + http_client: reqwest::Client, + cached_token: Arc>>, +} + +impl std::fmt::Debug for GoogleMetadataAuth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GoogleMetadataAuth") + .field("metadata_url", &self.config.metadata_url) + .field("scopes", &self.config.scopes) + .finish_non_exhaustive() + } +} + +impl GoogleMetadataAuth { + /// Create a new metadata-server auth provider. + pub fn new(config: GoogleMetadataConfig) -> Result { + let http_client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .timeout(std::time::Duration::from_secs(30)) + .build() + .map_err(|e| IcebergError::Http(format!("Failed to build HTTP client: {e}")))?; + + Ok(Self { + config, + http_client, + cached_token: Arc::new(RwLock::new(None)), + }) + } + + /// Fetch a fresh access token from the metadata server. + async fn fetch_token(&self) -> Result { + let response = self + .http_client + .get(&self.config.metadata_url) + .query(&[("scopes", self.config.scopes.as_str())]) + // The metadata server requires this header on every request. + .header("Metadata-Flavor", "Google") + .send() + .await + .map_err(|e| IcebergError::Http(format!("Metadata token request failed: {e}")))?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(IcebergError::Auth(format!( + "Google metadata token request failed ({status}): {body} \ + (the metadata server is only reachable on GCE/GKE; use a static bearer locally)" + ))); + } + + #[derive(serde::Deserialize)] + struct TokenResponse { + access_token: String, + #[serde(default = "default_token_type")] + token_type: String, + expires_in: Option, + } + + fn default_token_type() -> String { + "Bearer".to_string() + } + + let token_resp: TokenResponse = response.json().await.map_err(|e| { + IcebergError::Auth(format!("Failed to parse metadata token response: {e}")) + })?; + + // Default to 1 hour if not specified. + let expires_in = token_resp.expires_in.unwrap_or(3600); + let expires_at = Utc::now() + Duration::seconds(expires_in); + + tracing::debug!(expires_in = expires_in, "Google metadata token acquired"); + + Ok(CachedToken { + access_token: token_resp.access_token, + token_type: token_resp.token_type, + expires_at, + }) + } + + /// Get a valid token, refreshing if needed. + async fn get_token(&self) -> Result { + // Fast path: a cached, non-expired token. + { + let cached = self.cached_token.read().await; + if let Some(token) = cached.as_ref() { + if !token.is_expired() { + return Ok(token.clone()); + } + } + } + + // Slow path: refresh. + let new_token = self.fetch_token().await?; + { + let mut cached = self.cached_token.write().await; + *cached = Some(new_token.clone()); + } + Ok(new_token) + } +} + +#[async_trait(?Send)] +impl CatalogAuth for GoogleMetadataAuth { + async fn authorization_header(&self) -> Result> { + Ok(Some(self.get_token().await?.authorization_header())) + } + + async fn refresh(&self) -> Result<()> { + let new_token = self.fetch_token().await?; + *self.cached_token.write().await = Some(new_token); + Ok(()) + } +} + +#[async_trait] +impl SendCatalogAuth for GoogleMetadataAuth { + async fn authorization_header(&self) -> Result> { + Ok(Some(self.get_token().await?.authorization_header())) + } + + async fn refresh(&self) -> Result<()> { + let new_token = self.fetch_token().await?; + *self.cached_token.write().await = Some(new_token); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use wiremock::matchers::{header, method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn provider(url: String) -> GoogleMetadataAuth { + GoogleMetadataAuth::new(GoogleMetadataConfig { + metadata_url: url, + scopes: DEFAULT_SCOPES.to_string(), + }) + .unwrap() + } + + #[tokio::test] + async fn sends_flavor_header_and_scopes_and_caches() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/token")) + .and(header("metadata-flavor", "Google")) + .and(query_param( + "scopes", + "https://www.googleapis.com/auth/cloud-platform", + )) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/json") + .set_body_string( + r#"{"access_token":"ya29.abc","token_type":"Bearer","expires_in":3599}"#, + ), + ) + .mount(&server) + .await; + + let auth = provider(format!("{}/token", server.uri())); + let header = SendCatalogAuth::authorization_header(&auth).await.unwrap(); + assert_eq!(header, Some("Bearer ya29.abc".to_string())); + + // A second call reuses the cached (non-expired) token — no re-fetch. + SendCatalogAuth::authorization_header(&auth).await.unwrap(); + assert_eq!( + server.received_requests().await.unwrap().len(), + 1, + "token should be cached, not re-fetched" + ); + } + + #[tokio::test] + async fn surfaces_metadata_failure() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(403).set_body_string("Forbidden")) + .mount(&server) + .await; + + let auth = provider(format!("{}/token", server.uri())); + let err = SendCatalogAuth::authorization_header(&auth) + .await + .unwrap_err(); + assert!(err.to_string().contains("403"), "got: {err}"); + } +} diff --git a/fluree-db-iceberg/src/auth/mod.rs b/fluree-db-iceberg/src/auth/mod.rs index d83b2bbf4c..bbff15843a 100644 --- a/fluree-db-iceberg/src/auth/mod.rs +++ b/fluree-db-iceberg/src/auth/mod.rs @@ -8,11 +8,17 @@ //! //! - [`BearerTokenAuth`] - Static bearer token authentication //! - [`OAuth2ClientCredentials`] - OAuth2 client credentials flow with token caching +//! - [`GoogleMetadataAuth`] - Google metadata-server tokens (GKE Workload +//! Identity / GCE), refreshed automatically — for Google Iceberg REST catalogs +//! (BigLake), where a static bearer would expire mod bearer; +mod google_metadata; mod oauth2; +mod token; pub use bearer::BearerTokenAuth; +pub use google_metadata::{GoogleMetadataAuth, GoogleMetadataConfig}; pub use oauth2::{OAuth2ClientCredentials, OAuth2Config}; use crate::config_value::ConfigValue; @@ -82,6 +88,19 @@ pub enum AuthConfig { #[serde(default)] audience: Option, }, + /// Google metadata-server OAuth (GKE Workload Identity / GCE). Mints and + /// refreshes short-lived access tokens from the instance metadata server — + /// for Google Iceberg REST catalogs (BigLake), where a static bearer would + /// expire after ~1h with no way to renew. + #[serde(rename = "google_metadata")] + GoogleMetadata { + /// OAuth scopes (comma-separated). Defaults to cloud-platform. + #[serde(default)] + scopes: Option, + /// Token endpoint override (defaults to the metadata server). For tests. + #[serde(default)] + metadata_url: Option, + }, } impl AuthConfig { @@ -109,6 +128,13 @@ impl AuthConfig { }; Ok(Box::new(OAuth2ClientCredentials::new(config)?)) } + AuthConfig::GoogleMetadata { + scopes, + metadata_url, + } => Ok(Box::new(GoogleMetadataAuth::new(google_metadata_config( + scopes, + metadata_url, + ))?)), } } @@ -139,10 +165,32 @@ impl AuthConfig { }; Ok(std::sync::Arc::new(OAuth2ClientCredentials::new(config)?)) } + AuthConfig::GoogleMetadata { + scopes, + metadata_url, + } => Ok(std::sync::Arc::new(GoogleMetadataAuth::new( + google_metadata_config(scopes, metadata_url), + )?)), } } } +/// Build a [`GoogleMetadataConfig`], applying optional overrides over the +/// defaults (metadata-server URL + cloud-platform scope). +fn google_metadata_config( + scopes: &Option, + metadata_url: &Option, +) -> GoogleMetadataConfig { + let mut config = GoogleMetadataConfig::default(); + if let Some(scopes) = scopes { + config.scopes = scopes.clone(); + } + if let Some(metadata_url) = metadata_url { + config.metadata_url = metadata_url.clone(); + } + config +} + /// No-op authentication (for testing or public catalogs). #[derive(Debug)] pub struct NoAuth; @@ -192,6 +240,21 @@ mod tests { } } + #[test] + fn test_parse_google_metadata_auth() { + // Minimal form (no overrides) — defaults to the metadata server + cloud-platform. + let config: AuthConfig = serde_json::from_str(r#"{"type": "google_metadata"}"#).unwrap(); + assert!(matches!( + config, + AuthConfig::GoogleMetadata { + scopes: None, + metadata_url: None + } + )); + // Provider construction succeeds (no network I/O at build time). + assert!(config.create_provider_arc().is_ok()); + } + #[test] fn test_parse_oauth2_auth() { let json = r#"{ diff --git a/fluree-db-iceberg/src/auth/oauth2.rs b/fluree-db-iceberg/src/auth/oauth2.rs index 3a707d4277..af7ad265f3 100644 --- a/fluree-db-iceberg/src/auth/oauth2.rs +++ b/fluree-db-iceberg/src/auth/oauth2.rs @@ -1,10 +1,10 @@ //! OAuth2 client credentials flow authentication. +use crate::auth::token::CachedToken; use crate::auth::{CatalogAuth, SendCatalogAuth}; use crate::error::{IcebergError, Result}; use async_trait::async_trait; -use chrono::{DateTime, Duration, Utc}; -use rand::Rng; +use chrono::{Duration, Utc}; use std::sync::Arc; use tokio::sync::RwLock; @@ -18,31 +18,6 @@ pub struct OAuth2Config { pub audience: Option, } -/// Cached token with expiration. -#[derive(Debug, Clone)] -struct CachedToken { - access_token: String, - token_type: String, - expires_at: DateTime, -} - -impl CachedToken { - /// Check if token is expired or will expire within buffer period. - /// - /// Uses a 30-second base buffer plus 0-5s jitter to avoid thundering herds. - fn is_expired(&self) -> bool { - let jitter = rand::thread_rng().gen_range(0..5); - let buffer = Duration::seconds(30 + jitter); - Utc::now() + buffer >= self.expires_at - } - - /// Get the authorization header value. - fn authorization_header(&self) -> String { - // Use token_type from response (don't hardcode "Bearer") - format!("{} {}", self.token_type, self.access_token) - } -} - /// OAuth2 client credentials authentication. /// /// Handles token acquisition and automatic refresh before expiration. diff --git a/fluree-db-iceberg/src/auth/token.rs b/fluree-db-iceberg/src/auth/token.rs new file mode 100644 index 0000000000..27e48c8f8d --- /dev/null +++ b/fluree-db-iceberg/src/auth/token.rs @@ -0,0 +1,35 @@ +//! Shared cached access token with jittered expiry. +//! +//! Used by the refreshing catalog-auth providers ([`OAuth2ClientCredentials`] +//! and [`GoogleMetadataAuth`]) so the expiry/refresh policy lives in one place. +//! +//! [`OAuth2ClientCredentials`]: crate::auth::OAuth2ClientCredentials +//! [`GoogleMetadataAuth`]: crate::auth::GoogleMetadataAuth + +use chrono::{DateTime, Duration, Utc}; +use rand::Rng; + +/// A cached access token with its expiry. +#[derive(Debug, Clone)] +pub(crate) struct CachedToken { + pub(crate) access_token: String, + pub(crate) token_type: String, + pub(crate) expires_at: DateTime, +} + +impl CachedToken { + /// Check if the token is expired or will expire within the buffer period. + /// + /// Uses a 30-second base buffer plus 0-5s jitter to avoid thundering herds. + pub(crate) fn is_expired(&self) -> bool { + let jitter = rand::thread_rng().gen_range(0..5); + let buffer = Duration::seconds(30 + jitter); + Utc::now() + buffer >= self.expires_at + } + + /// Get the authorization header value (uses the response `token_type`, + /// rather than hardcoding `Bearer`). + pub(crate) fn authorization_header(&self) -> String { + format!("{} {}", self.token_type, self.access_token) + } +} diff --git a/fluree-db-server/src/routes/iceberg.rs b/fluree-db-server/src/routes/iceberg.rs index 494ca56997..1b10497e0b 100644 --- a/fluree-db-server/src/routes/iceberg.rs +++ b/fluree-db-server/src/routes/iceberg.rs @@ -47,6 +47,13 @@ pub struct IcebergMapRequest { pub oauth2_scope: Option, /// OAuth2 audience pub oauth2_audience: Option, + /// Use Google metadata-server auth (GKE Workload Identity / GCE) for the REST + /// catalog, minting + auto-refreshing short-lived tokens — for Google Iceberg + /// REST catalogs (BigLake), where a static `auth_bearer` would expire. + #[serde(default)] + pub auth_google_metadata: bool, + /// Optional OAuth scopes for `auth_google_metadata` (defaults to cloud-platform). + pub auth_google_scopes: Option, /// Warehouse identifier pub warehouse: Option, /// Disable vended credentials @@ -525,6 +532,11 @@ fn build_iceberg_config(req: &IcebergMapRequest) -> Result