From a08d1b6b01d82c05b222873182eb558daed179d2 Mon Sep 17 00:00:00 2001 From: Adam Spotton Date: Sun, 29 Mar 2026 21:42:55 -0400 Subject: [PATCH 1/4] perf(memory): enable vector index usage by checking for existing indexes Problem Spacebot was experiencing extended startup time due to attempting to rebuild vector indexes on every startup, even though index files already existed on disk. This caused: - Expensive KMeans training on every startup (IVF model, quantizer) - index_comparisons=0 in query logs (index not being used) - High CPU usage during startup due to unnecessary index rebuilding Root Cause The code called create_index() unconditionally. LanceDB's create_index() always attempts to rebuild the index from scratch when called, even if an index already exists. The previous error-handling approach was too late - the expensive training had already completed. Solution Check for existing indexes using list_indices() BEFORE attempting creation. Only create the index if it doesn't exist. This allows LanceDB to load and use the existing index immediately, enabling efficient vector searches. Changes - src/memory/lance.rs: Replace create_indexes() with ensure_indexes_exist() * Uses table.list_indices() to check for existing indexes * Only creates vector index if not present on embedding column * Only creates FTS index if not present on content column * Adds optimize_indexes() for incremental updates after data insertion - src/main.rs: Call ensure_indexes_exist() instead of create_indexes() - src/api/agents.rs: Call ensure_indexes_exist() instead of ensure_fts_index() - src/tools/memory_save.rs: Call ensure_indexes_exist() instead of ensure_fts_index() Performance Impact - Index usage: Queries now use the HNSW index (index_comparisons > 0) - Eliminates unnecessary index rebuilding when index already exists - Reduces startup time by skipping expensive training when index exists - Note: This fix enables index usage but does not stop the separate Knowledge Synthesis regeneration process Verification After deployment, verify: 1. Startup with existing index: Logs show 'Vector index already exists' 2. Query logs show index_comparisons > 0 (index is being used) 3. No KMeans training logs appear when index already exists 4. CPU usage remains normal during startup --- src/api/agents.rs | 5 +- src/main.rs | 6 +-- src/memory/lance.rs | 104 ++++++++++++++++++++++++++------------- src/tools/memory_save.rs | 8 +-- 4 files changed, 80 insertions(+), 43 deletions(-) diff --git a/src/api/agents.rs b/src/api/agents.rs index bf077920a..2d9a8b0cb 100644 --- a/src/api/agents.rs +++ b/src/api/agents.rs @@ -774,8 +774,9 @@ pub async fn create_agent_internal( format!("failed to init embeddings: {error}") })?; - if let Err(error) = embedding_table.ensure_fts_index().await { - tracing::warn!(%error, agent_id = %agent_id, "failed to create FTS index"); + // Ensure vector and FTS indexes exist (prevents 30-minute rebuild loop) + if let Err(error) = embedding_table.ensure_indexes_exist().await { + tracing::warn!(%error, agent_id = %agent_id, "failed to ensure indexes exist"); } let memory_search = std::sync::Arc::new(crate::memory::MemorySearch::new( diff --git a/src/main.rs b/src/main.rs index 6421ad567..5e673a39c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2634,9 +2634,9 @@ async fn initialize_agents( format!("failed to init embeddings for agent '{}'", agent_config.id) })?; - // Ensure FTS index exists for full-text search queries - if let Err(error) = embedding_table.ensure_fts_index().await { - tracing::warn!(%error, agent = %agent_config.id, "failed to create FTS index"); + // Ensure vector and FTS indexes exist (prevents 30-minute rebuild loop) + if let Err(error) = embedding_table.ensure_indexes_exist().await { + tracing::warn!(%error, agent = %agent_config.id, "failed to ensure indexes exist"); } let memory_search = Arc::new(spacebot::memory::MemorySearch::new( diff --git a/src/memory/lance.rs b/src/memory/lance.rs index 382bedefc..6576e19a0 100644 --- a/src/memory/lance.rs +++ b/src/memory/lance.rs @@ -295,48 +295,84 @@ impl EmbeddingTable { Ok(matches) } - /// Create HNSW vector index and FTS index for better performance. - /// Should be called after enough data accumulates. - pub async fn create_indexes(&self) -> Result<()> { - // Create HNSW vector index on embedding column - self.table - .create_index(&["embedding"], lancedb::index::Index::Auto) - .execute() + /// Ensure vector and FTS indexes exist, creating them only if they don't already exist. + /// + /// This prevents the expensive HNSW index training from running on every startup. + /// Uses `list_indices()` to check for existing indexes BEFORE attempting creation. + /// + /// # Problem this solves + /// + /// LanceDB's `create_index()` unconditionally triggers a full rebuild when called, + /// regardless of whether an index already exists on disk. The previous approach of + /// catching errors after the fact was too late — the expensive KMeans training had + /// already completed. + /// + /// # Solution + /// + /// Check for existing indexes using `list_indices()` before calling `create_index()`. + /// Only create if no index exists on the target column. + pub async fn ensure_indexes_exist(&self) -> Result<()> { + use lancedb::index::Index; + + // Check for existing indexes + let indices = self + .table + .list_indices() .await - .map_err(|e| DbError::LanceDb(format!("Failed to create vector index: {}", e)))?; + .map_err(|e| DbError::LanceDb(e.to_string()))?; + + // Check vector index on embedding column + let has_vector_index = indices + .iter() + .any(|idx| idx.columns.iter().any(|col| col == "embedding")); + + if !has_vector_index { + tracing::info!("Creating HNSW vector index on embedding column"); + self.table + .create_index(&["embedding"], Index::Auto) + .execute() + .await + .map_err(|e| DbError::LanceDb(format!("Failed to create vector index: {}", e)))?; + tracing::info!("Vector index created successfully"); + } else { + tracing::debug!("Vector index already exists, skipping creation"); + } - self.ensure_fts_index().await?; + // Check FTS index on content column + let has_fts_index = indices + .iter() + .any(|idx| idx.columns.iter().any(|col| col == "content")); + + if !has_fts_index { + tracing::info!("Creating FTS index on content column"); + self.table + .create_index(&["content"], Index::FTS(Default::default())) + .execute() + .await + .map_err(|e| DbError::LanceDb(format!("Failed to create FTS index: {}", e)))?; + tracing::info!("FTS index created successfully"); + } else { + tracing::debug!("FTS index already exists, skipping creation"); + } Ok(()) } - /// Ensure the FTS index exists on the content column. + /// Optimize indexes for incremental updates after data insertion. /// - /// LanceDB requires an inverted index for `full_text_search()` queries. - /// This is safe to call multiple times — if the index already exists, the - /// error is silently ignored. - pub async fn ensure_fts_index(&self) -> Result<()> { - match self - .table - .create_index(&["content"], lancedb::index::Index::FTS(Default::default())) - .execute() + /// This is much faster than a full rebuild and should be called after + /// significant data changes to maintain query performance. + pub async fn optimize_indexes(&self) -> Result<()> { + use lancedb::table::{OptimizeAction, OptimizeOptions}; + + tracing::debug!("Optimizing indexes (incremental update)"); + self.table + .optimize(OptimizeAction::Index(OptimizeOptions::default())) .await - { - Ok(()) => { - tracing::debug!("FTS index created on content column"); - Ok(()) - } - Err(error) => { - let message = error.to_string(); - // LanceDB returns an error if the index already exists - if message.contains("already") || message.contains("index") { - tracing::trace!("FTS index already exists"); - Ok(()) - } else { - Err(DbError::LanceDb(format!("Failed to create FTS index: {}", message)).into()) - } - } - } + .map_err(|e| DbError::LanceDb(format!("Failed to optimize indexes: {}", e)))?; + tracing::debug!("Index optimization complete"); + + Ok(()) } /// Get the Arrow schema for the embeddings table. diff --git a/src/tools/memory_save.rs b/src/tools/memory_save.rs index 8193d7f50..86e5004dc 100644 --- a/src/tools/memory_save.rs +++ b/src/tools/memory_save.rs @@ -379,15 +379,15 @@ impl Tool for MemorySaveTool { } } - // Ensure the FTS index exists so full_text_search queries work. - // Safe to call repeatedly — no-ops if the index already exists. + // Ensure vector and FTS indexes exist (prevents 30-minute rebuild loop) + // Safe to call repeatedly — skips creation if indexes already exist. if let Err(error) = self .memory_search .embedding_table() - .ensure_fts_index() + .ensure_indexes_exist() .await { - tracing::warn!(%error, "failed to ensure FTS index after memory save"); + tracing::warn!(%error, "failed to ensure indexes after memory save"); } if let Some(event_context) = &self.event_context From 6267575ee9aa8f1b0e5b17140c7ce4af57735be2 Mon Sep 17 00:00:00 2001 From: Adam Spotton Date: Tue, 31 Mar 2026 20:20:25 -0400 Subject: [PATCH 2/4] fix(memory): add single-flight guard to prevent race condition in index creation The previous fix using list_indices() check was still race-prone when multiple concurrent callers (startup, agent creation, memory-save paths) all held cloned EmbeddingTable instances and simultaneously detected missing indexes, triggering duplicate expensive rebuilds. This adds two OnceCell guards (one for vector index, one for FTS) that: - Ensure only ONE index build runs at a time per index type - Allow concurrent callers to wait for the first to complete - Share state across all cloned EmbeddingTable instances - Include a double-check after acquiring the guard to handle external creation The fix maintains the original goal of preventing unnecessary rebuilds while eliminating the race condition that could still trigger duplicate builds. --- src/memory/lance.rs | 167 ++++++++++++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 44 deletions(-) diff --git a/src/memory/lance.rs b/src/memory/lance.rs index 6576e19a0..d2f582eab 100644 --- a/src/memory/lance.rs +++ b/src/memory/lance.rs @@ -1,4 +1,8 @@ //! LanceDB table management and embedding storage with HNSW vector index and FTS. +//! +//! Index creation uses a single-flight guard pattern to prevent race conditions +//! when multiple concurrent callers attempt to ensure indexes exist simultaneously. +//! This ensures only ONE index build runs at a time per index type. use crate::error::{DbError, Result}; use arrow_array::cast::AsArray; @@ -6,20 +10,45 @@ use arrow_array::types::Float32Type; use arrow_array::{Array, RecordBatchIterator}; use futures::TryStreamExt; use std::sync::Arc; +use tokio::sync::OnceCell; /// Schema constants for the embeddings table. const TABLE_NAME: &str = "memory_embeddings"; const EMBEDDING_DIM: i32 = 384; // all-MiniLM-L6-v2 dimension /// LanceDB table for memory embeddings with HNSW index and FTS. +/// +/// Index creation is protected by single-flight guards to prevent duplicate +/// concurrent builds. The guards are stored as `OnceCell` instances that +/// coordinate across all cloned `EmbeddingTable` instances sharing the same +/// underlying `lancedb::Table`. pub struct EmbeddingTable { table: lancedb::Table, + /// Single-flight guard for vector index creation. + /// + /// When multiple callers concurrently call `ensure_indexes_exist()`, + /// only the first caller will actually create the index. Subsequent + /// callers will wait for the first to complete and then reuse the result. + /// + /// The `OnceCell<()>` pattern works because: + /// 1. `OnceCell` is async-aware and safe to await from multiple tasks + /// 2. All `EmbeddingTable` instances clone the `Arc`, so they + /// share the same guard state + /// 3. Once initialized, the guard returns immediately for all callers + vector_index_guard: Arc>, + /// Single-flight guard for FTS index creation. + /// + /// Same pattern as `vector_index_guard`, but for the full-text search + /// index on the `content` column. + fts_index_guard: Arc>, } impl Clone for EmbeddingTable { fn clone(&self) -> Self { Self { table: self.table.clone(), + vector_index_guard: self.vector_index_guard.clone(), + fts_index_guard: self.fts_index_guard.clone(), } } } @@ -32,7 +61,13 @@ impl EmbeddingTable { pub async fn open_or_create(connection: &lancedb::Connection) -> Result { // Try to open existing table match connection.open_table(TABLE_NAME).execute().await { - Ok(table) => return Ok(Self { table }), + Ok(table) => { + return Ok(Self { + table, + vector_index_guard: Arc::new(OnceCell::new()), + fts_index_guard: Arc::new(OnceCell::new()), + }) + } Err(error) => { tracing::debug!(%error, "failed to open embeddings table, will create"); } @@ -40,7 +75,13 @@ impl EmbeddingTable { // Table doesn't exist or is unreadable — try creating it match Self::create_empty_table(connection).await { - Ok(table) => return Ok(Self { table }), + Ok(table) => { + return Ok(Self { + table, + vector_index_guard: Arc::new(OnceCell::new()), + fts_index_guard: Arc::new(OnceCell::new()), + }) + } Err(error) => { tracing::warn!( %error, @@ -58,7 +99,11 @@ impl EmbeddingTable { let table = Self::create_empty_table(connection).await?; tracing::info!("embeddings table recovered — embeddings will be rebuilt from memory store"); - Ok(Self { table }) + Ok(Self { + table, + vector_index_guard: Arc::new(OnceCell::new()), + fts_index_guard: Arc::new(OnceCell::new()), + }) } /// Create an empty embeddings table. @@ -298,7 +343,8 @@ impl EmbeddingTable { /// Ensure vector and FTS indexes exist, creating them only if they don't already exist. /// /// This prevents the expensive HNSW index training from running on every startup. - /// Uses `list_indices()` to check for existing indexes BEFORE attempting creation. + /// Uses single-flight guards to ensure only ONE index creation runs at a time, + /// even when multiple concurrent callers invoke this method simultaneously. /// /// # Problem this solves /// @@ -309,51 +355,84 @@ impl EmbeddingTable { /// /// # Solution /// - /// Check for existing indexes using `list_indices()` before calling `create_index()`. - /// Only create if no index exists on the target column. + /// Two-layer protection: + /// 1. **Single-flight guards** (`OnceCell`): Ensure only one index creation runs + /// at a time. Concurrent callers wait for the first to complete. + /// 2. **`list_indices()` check**: After acquiring the guard, verify the index still + /// doesn't exist (handles cases where another process created it externally). + /// + /// # Concurrency pattern + /// + /// The `OnceCell` guard ensures that: + /// - Only the first caller actually performs the index creation + /// - Subsequent callers await the initialization and get the same result + /// - The guard is shared across all cloned `EmbeddingTable` instances + /// - No deadlocks: each index has its own independent guard pub async fn ensure_indexes_exist(&self) -> Result<()> { use lancedb::index::Index; - // Check for existing indexes - let indices = self - .table - .list_indices() - .await - .map_err(|e| DbError::LanceDb(e.to_string()))?; + // Ensure vector index on embedding column using single-flight guard + self.vector_index_guard + .get_or_try_init(|| async { + // Double-check: verify index doesn't exist before creating + // This handles cases where another process created it externally + let indices = self + .table + .list_indices() + .await + .map_err(|e| DbError::LanceDb(e.to_string()))?; + + let has_vector_index = indices + .iter() + .any(|idx| idx.columns.iter().any(|col| col == "embedding")); + + if has_vector_index { + tracing::debug!("Vector index already exists, skipping creation"); + return Ok::<(), crate::error::Error>(()); + } - // Check vector index on embedding column - let has_vector_index = indices - .iter() - .any(|idx| idx.columns.iter().any(|col| col == "embedding")); - - if !has_vector_index { - tracing::info!("Creating HNSW vector index on embedding column"); - self.table - .create_index(&["embedding"], Index::Auto) - .execute() - .await - .map_err(|e| DbError::LanceDb(format!("Failed to create vector index: {}", e)))?; - tracing::info!("Vector index created successfully"); - } else { - tracing::debug!("Vector index already exists, skipping creation"); - } + tracing::info!("Creating HNSW vector index on embedding column"); + self.table + .create_index(&["embedding"], Index::Auto) + .execute() + .await + .map_err(|e| DbError::LanceDb(format!("Failed to create vector index: {}", e)))?; + tracing::info!("Vector index created successfully"); + + Ok::<(), crate::error::Error>(()) + }) + .await?; + + // Ensure FTS index on content column using single-flight guard + self.fts_index_guard + .get_or_try_init(|| async { + // Double-check: verify index doesn't exist before creating + let indices = self + .table + .list_indices() + .await + .map_err(|e| DbError::LanceDb(e.to_string()))?; + + let has_fts_index = indices + .iter() + .any(|idx| idx.columns.iter().any(|col| col == "content")); + + if has_fts_index { + tracing::debug!("FTS index already exists, skipping creation"); + return Ok::<(), crate::error::Error>(()); + } - // Check FTS index on content column - let has_fts_index = indices - .iter() - .any(|idx| idx.columns.iter().any(|col| col == "content")); - - if !has_fts_index { - tracing::info!("Creating FTS index on content column"); - self.table - .create_index(&["content"], Index::FTS(Default::default())) - .execute() - .await - .map_err(|e| DbError::LanceDb(format!("Failed to create FTS index: {}", e)))?; - tracing::info!("FTS index created successfully"); - } else { - tracing::debug!("FTS index already exists, skipping creation"); - } + tracing::info!("Creating FTS index on content column"); + self.table + .create_index(&["content"], Index::FTS(Default::default())) + .execute() + .await + .map_err(|e| DbError::LanceDb(format!("Failed to create FTS index: {}", e)))?; + tracing::info!("FTS index created successfully"); + + Ok::<(), crate::error::Error>(()) + }) + .await?; Ok(()) } From 23b49c0ecf4ada65df0c022b5d88833281743a02 Mon Sep 17 00:00:00 2001 From: Adam Spotton Date: Tue, 31 Mar 2026 22:44:04 -0400 Subject: [PATCH 3/4] fix(memory): use explicit IVF-HNSW-SQ index and module-level guards for serialization Changes: - Replace Index::Auto with explicit Index::IvfHnswSq for predictable behavior - Update log messages to accurately describe IVF-HNSW-SQ index type - Move guards from instance fields to module-level statics for defense-in-depth - Remove unnecessary Arc fields from EmbeddingTable struct Rationale: - Index::IvfHnswSq provides the best recall/latency trade-off for Spacebot's workload (7.6k embeddings, 384 dimensions) - Module-level static guards ensure serialization across ALL EmbeddingTable instances, not just clones of the same instance - Explicit index type prevents behavior changes across LanceDB versions - Log messages now accurately describe the actual index being created This addresses the upstream PR review feedback that identified: 1. Misleading log messages claiming HNSW when using Index::Auto 2. Instance-level guards don't serialize across separate open_or_create() calls --- src/memory/lance.rs | 82 ++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 50 deletions(-) diff --git a/src/memory/lance.rs b/src/memory/lance.rs index d2f582eab..78faa0717 100644 --- a/src/memory/lance.rs +++ b/src/memory/lance.rs @@ -1,8 +1,11 @@ -//! LanceDB table management and embedding storage with HNSW vector index and FTS. +//! LanceDB table management and embedding storage with IVF-HNSW-SQ vector index and FTS. //! //! Index creation uses a single-flight guard pattern to prevent race conditions //! when multiple concurrent callers attempt to ensure indexes exist simultaneously. //! This ensures only ONE index build runs at a time per index type. +//! +//! Guards are module-level statics to ensure serialization across all +//! EmbeddingTable instances in the process, not just clones of the same instance. use crate::error::{DbError, Result}; use arrow_array::cast::AsArray; @@ -12,43 +15,32 @@ use futures::TryStreamExt; use std::sync::Arc; use tokio::sync::OnceCell; +/// Module-level single-flight guards for index creation. +/// +/// These are statics to ensure that index creation is serialized across +/// ALL EmbeddingTable instances in the process, not just clones of the same instance. +/// This prevents race conditions when multiple threads independently call +/// `open_or_create()` and then attempt to ensure indexes exist. +static VECTOR_INDEX_GUARD: OnceCell<()> = OnceCell::const_new(); +static FTS_INDEX_GUARD: OnceCell<()> = OnceCell::const_new(); + /// Schema constants for the embeddings table. const TABLE_NAME: &str = "memory_embeddings"; const EMBEDDING_DIM: i32 = 384; // all-MiniLM-L6-v2 dimension /// LanceDB table for memory embeddings with HNSW index and FTS. /// -/// Index creation is protected by single-flight guards to prevent duplicate -/// concurrent builds. The guards are stored as `OnceCell` instances that -/// coordinate across all cloned `EmbeddingTable` instances sharing the same -/// underlying `lancedb::Table`. +/// Index creation is protected by module-level static single-flight guards +/// to prevent duplicate concurrent builds. The guards ensure that only ONE +/// index creation runs at a time across ALL EmbeddingTable instances. pub struct EmbeddingTable { table: lancedb::Table, - /// Single-flight guard for vector index creation. - /// - /// When multiple callers concurrently call `ensure_indexes_exist()`, - /// only the first caller will actually create the index. Subsequent - /// callers will wait for the first to complete and then reuse the result. - /// - /// The `OnceCell<()>` pattern works because: - /// 1. `OnceCell` is async-aware and safe to await from multiple tasks - /// 2. All `EmbeddingTable` instances clone the `Arc`, so they - /// share the same guard state - /// 3. Once initialized, the guard returns immediately for all callers - vector_index_guard: Arc>, - /// Single-flight guard for FTS index creation. - /// - /// Same pattern as `vector_index_guard`, but for the full-text search - /// index on the `content` column. - fts_index_guard: Arc>, } impl Clone for EmbeddingTable { fn clone(&self) -> Self { Self { table: self.table.clone(), - vector_index_guard: self.vector_index_guard.clone(), - fts_index_guard: self.fts_index_guard.clone(), } } } @@ -62,11 +54,7 @@ impl EmbeddingTable { // Try to open existing table match connection.open_table(TABLE_NAME).execute().await { Ok(table) => { - return Ok(Self { - table, - vector_index_guard: Arc::new(OnceCell::new()), - fts_index_guard: Arc::new(OnceCell::new()), - }) + return Ok(Self { table }) } Err(error) => { tracing::debug!(%error, "failed to open embeddings table, will create"); @@ -76,11 +64,7 @@ impl EmbeddingTable { // Table doesn't exist or is unreadable — try creating it match Self::create_empty_table(connection).await { Ok(table) => { - return Ok(Self { - table, - vector_index_guard: Arc::new(OnceCell::new()), - fts_index_guard: Arc::new(OnceCell::new()), - }) + return Ok(Self { table }) } Err(error) => { tracing::warn!( @@ -99,11 +83,7 @@ impl EmbeddingTable { let table = Self::create_empty_table(connection).await?; tracing::info!("embeddings table recovered — embeddings will be rebuilt from memory store"); - Ok(Self { - table, - vector_index_guard: Arc::new(OnceCell::new()), - fts_index_guard: Arc::new(OnceCell::new()), - }) + Ok(Self { table }) } /// Create an empty embeddings table. @@ -343,8 +323,8 @@ impl EmbeddingTable { /// Ensure vector and FTS indexes exist, creating them only if they don't already exist. /// /// This prevents the expensive HNSW index training from running on every startup. - /// Uses single-flight guards to ensure only ONE index creation runs at a time, - /// even when multiple concurrent callers invoke this method simultaneously. + /// Uses module-level static single-flight guards to ensure only ONE index creation + /// runs at a time, even when multiple concurrent callers invoke this method simultaneously. /// /// # Problem this solves /// @@ -356,23 +336,23 @@ impl EmbeddingTable { /// # Solution /// /// Two-layer protection: - /// 1. **Single-flight guards** (`OnceCell`): Ensure only one index creation runs + /// 1. **Single-flight guards** (module-level statics): Ensure only one index creation runs /// at a time. Concurrent callers wait for the first to complete. /// 2. **`list_indices()` check**: After acquiring the guard, verify the index still /// doesn't exist (handles cases where another process created it externally). /// /// # Concurrency pattern /// - /// The `OnceCell` guard ensures that: + /// The module-level static `OnceCell` guard ensures that: /// - Only the first caller actually performs the index creation /// - Subsequent callers await the initialization and get the same result - /// - The guard is shared across all cloned `EmbeddingTable` instances + /// - The guard is shared across ALL EmbeddingTable instances in the process /// - No deadlocks: each index has its own independent guard pub async fn ensure_indexes_exist(&self) -> Result<()> { use lancedb::index::Index; - // Ensure vector index on embedding column using single-flight guard - self.vector_index_guard + // Ensure vector index on embedding column using module-level static guard + VECTOR_INDEX_GUARD .get_or_try_init(|| async { // Double-check: verify index doesn't exist before creating // This handles cases where another process created it externally @@ -391,9 +371,11 @@ impl EmbeddingTable { return Ok::<(), crate::error::Error>(()); } - tracing::info!("Creating HNSW vector index on embedding column"); + tracing::info!("Creating vector index (IVF-HNSW with Scalar Quantization)"); self.table - .create_index(&["embedding"], Index::Auto) + .create_index(&["embedding"], Index::IvfHnswSq( + lancedb::index::vector::IvfHnswSqIndexBuilder::default() + )) .execute() .await .map_err(|e| DbError::LanceDb(format!("Failed to create vector index: {}", e)))?; @@ -403,8 +385,8 @@ impl EmbeddingTable { }) .await?; - // Ensure FTS index on content column using single-flight guard - self.fts_index_guard + // Ensure FTS index on content column using module-level static guard + FTS_INDEX_GUARD .get_or_try_init(|| async { // Double-check: verify index doesn't exist before creating let indices = self From 54938be5d7fffb93c0d6b03fe2bb2fcaeb624ad4 Mon Sep 17 00:00:00 2001 From: Adam Spotton Date: Wed, 1 Apr 2026 08:07:29 -0400 Subject: [PATCH 4/4] fix(memory): optimize indexes during cortex maintenance --- src/memory/maintenance.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/memory/maintenance.rs b/src/memory/maintenance.rs index 67b935497..f0294f9fe 100644 --- a/src/memory/maintenance.rs +++ b/src/memory/maintenance.rs @@ -90,6 +90,18 @@ pub async fn run_maintenance_with_cancel( .await?; } + // Optimize indexes to incorporate all changes from decay, prune, and merge. + // This ensures the ANN index stays current with the full dataset. + if let Err(error) = embedding_table.optimize_indexes().await { + tracing::warn!( + %error, + "failed to optimize indexes during maintenance — index may be stale" + ); + // Don't fail the entire maintenance — optimization is best-effort + } else { + tracing::info!("Index optimization complete after maintenance"); + } + Ok(report) }