feat: promote RankQuantFastscan to public API + .ovfs persistence

fuzz/Cargo.toml

@@ -49,6 +49,13 @@ test = false
 doc = false
 bench = false
 
+[[bin]]
+name = "load_fastscan"
+path = "fuzz_targets/load_fastscan.rs"
+test = false
+doc = false
+bench = false
+
 # Hot-path targets (beyond the loaders): the ingest + asymmetric-search compute
 # path, and the write -> load round-trip the loader targets never reach.
 [[bin]]

fuzz/fuzz_targets/load_fastscan.rs

@@ -0,0 +1,27 @@
+//! libFuzzer target for the `.ovfs` / `OVFS` loader (the FastScan b=2
+//! persistence format — new in the ordvec format, no legacy `TV*` magic),
+//! driven through the public `ordvec::RankQuantFastscan::load` entry point.
+//!
+//! The low-level `rank_io::load_fastscan` parser is crate-internal
+//! (`pub(crate)`), so the fuzzer exercises it through `RankQuantFastscan::load`
+//! — which runs that exact loader (the full public load path). `load` takes a
+//! `&Path` and the only public load entry points are path-based (issue #6), so
+//! a shared process-local scratch file (see [`scratch`]) feeds the loader the
+//! fuzz bytes without per-iteration `mkstemp`/`unlink` churn.
+//!
+//! Contract: on arbitrary bytes the loader must return `Ok(..)` or `Err(..)` —
+//! never panic, abort, or read out of bounds. libFuzzer treats any panic/abort
+//! as a crash, so simply letting the result drop is the assertion.
+
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+
+mod scratch;
+
+fuzz_target!(|data: &[u8]| {
+    scratch::with_scratch_file(data, |path| {
+        // The only thing under test: arbitrary bytes -> Ok | Err, no panic.
+        let _ = ordvec::RankQuantFastscan::load(path);
+    });
+});

src/fastscan.rs

@@ -511,21 +511,22 @@ pub(crate) fn search_asymmetric_fastscan_b2(
 ///   cleanly with incremental extend (tail padding within blocks
 ///   would interleave with new docs). Subsequent `add()` calls panic;
 ///   construct a new index for incremental scenarios.
-/// - **No `swap_remove`, `write`, `load`** — the block-32 layout
-///   makes byte-exact updates non-trivial. v2 follow-up.
+/// - **No `swap_remove`** — the block-32 layout makes byte-exact in-place
+///   updates non-trivial (a v2 follow-up). Persistence *is* supported:
+///   [`write`](Self::write) / [`load`](Self::load) round-trip via the
+///   `.ovfs` format.
 ///
 /// # Concurrency
 ///
 /// `search` takes `&self`; safe to call from multiple threads
 /// concurrently.
 ///
-/// # Visibility
+/// # Positioning
 ///
-/// This type is re-exported `#[doc(hidden)]`: it is an optional scan
-/// path, not part of the headline API. Prefer
-/// [`RankQuant`](crate::RankQuant) unless you have
-/// measured FastScan to win on your workload.
-#[doc(hidden)]
+/// A stable, documented public type, but a **specialized** one: it is the
+/// minimum-latency b=2 scan path, not the headline retrieval API. Prefer
+/// [`RankQuant`](crate::RankQuant) / [`Bitmap`](crate::Bitmap) / the two-stage
+/// flow unless you have measured FastScan to win on your workload.
 pub struct RankQuantFastscan {
     dim: usize,
     n_vectors: usize,
@@ -640,4 +641,29 @@ impl RankQuantFastscan {
     pub fn byte_size(&self) -> usize {
         self.packed_fs.len()
     }
+
+    /// Persist this index to a `.ovfs` file (magic `OVFS`).
+    ///
+    /// The on-disk form is a 13-byte header (`OVFS` magic, version, `dim`,
+    /// `n_vectors`) followed by the opaque block-32 packed FastScan payload.
+    /// This is a new ordvec format with no turbovec-era counterpart. Round-trip
+    /// is a type-level guarantee: [`Self::load`] reconstructs the same
+    /// `(dim, n_vectors)` and packed buffer this writes.
+    pub fn write(&self, path: impl AsRef<std::path::Path>) -> std::io::Result<()> {
+        crate::rank_io::write_fastscan(path, self.dim, self.n_vectors, &self.packed_fs)
+    }
+
+    /// Load a `.ovfs` FastScan index previously written by [`Self::write`].
+    ///
+    /// The loader validates the header and that the payload length is exactly
+    /// the block-32 size implied by `(dim, n_vectors)` (`dim % 4 == 0`, no
+    /// trailing bytes), so the returned index is consistent by construction.
+    pub fn load(path: impl AsRef<std::path::Path>) -> std::io::Result<Self> {
+        let (dim, n_vectors, packed_fs) = crate::rank_io::load_fastscan(path)?;
+        Ok(Self {
+            dim,
+            n_vectors,
+            packed_fs,
+        })
+    }
 }

src/lib.rs

@@ -27,6 +27,12 @@
 //!   coordinate, set when the coordinate is positive) for sign-cosine
 //!   candidate generation.
 //!
+//! For b=2 specifically, [`RankQuantFastscan`] is a specialized companion to
+//! [`RankQuant`] — a block-32 FastScan kernel (nibble LUT; AVX-512 → AVX2 →
+//! scalar dispatch) for absolute-minimum stage-1 scan latency, trading 2× the
+//! b=2 storage and 8-bit LUT scoring noise. Reach for it only when scan latency
+//! is the binding constraint.
+//!
 //! These four families are the retrieval surface. The `experimental`
 //! `MultiBucketBitmap` indexed contingency / projection API is a niche
 //! research/analysis substrate for the bilinear bucket-overlap decomposition —
@@ -162,11 +168,12 @@ pub use const_weight_bitmap::{
     choose, top_group_overlap_vector, BitmapNull, ConstantWeightBitmap, PackedConstantWeightBitmap,
 };
 
-// `RankQuantFastscan` is an optional FastScan b=2 scan path. It is
-// re-exported `#[doc(hidden)]` at the crate root — reachable as
-// `ordvec::RankQuantFastscan` for callers who opt in, but not
-// advertised alongside the headline index types above.
-#[doc(hidden)]
+// `RankQuantFastscan` is a specialized b=2 FastScan scan path (block-32 nibble
+// LUT, AVX-512 → AVX2 → scalar dispatch) for absolute-minimum stage-1 scan
+// latency, at the cost of 2× the `RankQuant` b=2 storage and 8-bit LUT scoring
+// noise. It is a stable, documented public type, but a *specialized* one — the
+// headline retrieval surface is still `RankQuant` / `Bitmap` / two-stage; reach
+// for FastScan only when scan latency at b=2 is the binding constraint.
 pub use fastscan::RankQuantFastscan;
 
 /// Whether the AVX-512 VPOPCNTDQ bitmap/sign scan kernels are active on this

src/rank_io.rs

@@ -1,13 +1,15 @@
 //! Read/write ordinal/sign index files.
 //!
-//! Four formats live here, each self-describing via a 4-byte magic. Files
+//! Five formats live here, each self-describing via a 4-byte magic. Files
 //! written by this crate use the **`.ov*` / `OV*`** magics (the ordvec format);
 //! the legacy turbovec-era **`.tv*` / `TV*`** magics are still accepted on load
 //! for backward compatibility, but are never written:
 //! * `.ovr`  (legacy `.tvr`)  — [`Rank`](crate::Rank) — magic `OVR1` (also reads `TVR1`)
 //! * `.ovrq` (legacy `.tvrq`) — [`RankQuant`](crate::RankQuant) — magic `OVRQ` (also reads `TVRQ`)
 //! * `.ovbm` (legacy `.tvbm`) — [`Bitmap`](crate::Bitmap) — magic `OVBM` (also reads `TVBM`)
 //! * `.ovsb` (legacy `.tvsb`) — [`SignBitmap`](crate::SignBitmap) — magic `OVSB` (also reads `TVSB`)
+//! * `.ovfs` — [`RankQuantFastscan`](crate::RankQuantFastscan) — magic `OVFS`
+//!   (new in the ordvec format; no legacy counterpart)
 //!
 //! All formats are little-endian. Headers are small fixed-size structs
 //! followed by a single contiguous payload (the rank / packed / bitmap
@@ -67,6 +69,9 @@ const OVR_MAGIC: &[u8; 4] = b"OVR1";
 const OVRQ_MAGIC: &[u8; 4] = b"OVRQ";
 const OVBM_MAGIC: &[u8; 4] = b"OVBM";
 const OVSB_MAGIC: &[u8; 4] = b"OVSB";
+// FastScan b=2 block-32 layout (`RankQuantFastscan`). New in the ordvec format —
+// there is no turbovec-era counterpart, so it has no legacy magic.
+const OVFS_MAGIC: &[u8; 4] = b"OVFS";
 // Legacy turbovec-era magics — still accepted on load for backward
 // compatibility, never written. Files produced before the ordvec rebrand carry
 // these; loaders accept either the `OV*` or the matching `TV*` magic.
@@ -221,7 +226,7 @@ fn check_dim(dim: usize) -> io::Result<()> {
     Ok(())
 }
 
-/// Dimension check for `.tvsb` sign-bitmap files.
+/// Dimension check for `.ovsb` sign-bitmap files.
 ///
 /// The `u16::MAX` ceiling in [`check_dim`] exists to honour
 /// [`crate::Rank`]'s `u16` rank-storage invariant. Sign bitmaps
@@ -772,7 +777,7 @@ pub(crate) fn load_bitmap(path: impl AsRef<Path>) -> io::Result<(usize, usize, u
     Ok((dim, n_top, n_vectors, bitmaps))
 }
 
-/// Persist a [`crate::SignBitmap`] payload to a `.tvsb` file.
+/// Persist a [`crate::SignBitmap`] payload to a `.ovsb` file.
 ///
 /// On-disk layout (little-endian throughout):
 ///
@@ -810,7 +815,7 @@ pub(crate) fn write_sign_bitmap(
     Ok(())
 }
 
-/// Load a `.tvsb` file written by `write_sign_bitmap`.
+/// Load a `.ovsb` file written by `write_sign_bitmap`.
 ///
 /// Validates magic, version, dim (must be in
 /// `[64, MAX_SIGN_BITMAP_DIM]` and a multiple of 64), and `n_vectors`
@@ -858,6 +863,95 @@ pub(crate) fn load_sign_bitmap(path: impl AsRef<Path>) -> io::Result<(usize, usi
     Ok((dim, n_vectors, bitmaps))
 }
 
+// -------------------------------------------------------------------
+// RankQuantFastscan: b=2 block-32 FastScan layout.
+// Header: magic(4) | version(1) | dim(u32 LE) | n_vectors(u32 LE) = 13 B
+// Payload: n_blocks * (dim/2) * 32 bytes, n_blocks = ceil(n_vectors / 32).
+// New ordvec format (no legacy TV* counterpart).
+// -------------------------------------------------------------------
+
+fn fastscan_payload_bytes(dim: usize, vector_count: usize) -> io::Result<usize> {
+    // FastScan b=2 packs 32 docs per block; each block holds `pairs * 32` bytes
+    // (`pairs = dim / 2`). `dim % 4 == 0` is enforced by the loader / constructor
+    // before this is called, so `dim / 2` is exact. An empty corpus has zero
+    // blocks and zero payload.
+    let n_blocks = vector_count.div_ceil(32);
+    let pairs = dim / 2;
+    n_blocks
+        .checked_mul(pairs)
+        .and_then(|x| x.checked_mul(32))
+        .ok_or_else(|| invalid("OVFS payload size overflows usize"))
+}
+
+pub(crate) fn write_fastscan(
+    path: impl AsRef<Path>,
+    dim: usize,
+    n_vectors: usize,
+    packed_fs: &[u8],
+) -> io::Result<()> {
+    // Validate every header parameter *before* File::create, so a now-public
+    // persistence API never (a) silently truncates `dim`/`n_vectors` through the
+    // `as u32` casts below, (b) writes a corrupt/oversized file (the loaders'
+    // MAX_PAYLOAD cap; a rejected write never truncates an existing file), or
+    // (c) panics from a `Result`-returning fn. Mirrors load_fastscan's contract.
+    check_dim(dim)?;
+    if !dim.is_multiple_of(4) {
+        return Err(invalid(format!(
+            "OVFS dim {dim} is not a multiple of 4 (FastScan b=2 constant composition)"
+        )));
+    }
+    check_n_vectors(n_vectors)?;
+    let payload_bytes = fastscan_payload_bytes(dim, n_vectors)?;
+    check_payload_bytes(payload_bytes)?;
+    if packed_fs.len() != payload_bytes {
+        return Err(invalid(format!(
+            "OVFS packed buffer is {} bytes but dim={dim}/n_vectors={n_vectors} implies {payload_bytes}",
+            packed_fs.len()
+        )));
+    }
+    let mut f = BufWriter::new(File::create(path)?);
+    f.write_all(OVFS_MAGIC)?;
+    f.write_all(&[VERSION])?;
+    f.write_all(&(dim as u32).to_le_bytes())?;
+    f.write_all(&(n_vectors as u32).to_le_bytes())?;
+    f.write_all(packed_fs)?;
+    f.flush()?;
+    Ok(())
+}
+
+pub(crate) fn load_fastscan(path: impl AsRef<Path>) -> io::Result<(usize, usize, Vec<u8>)> {
+    let file = File::open(path)?;
+    let file_len = file.metadata()?.len();
+    let mut f = BufReader::new(file);
+    let magic = read_magic(&mut f, "OVFS")?;
+    // OVFS is new in the ordvec format: there is no legacy TV* fastscan magic.
+    if &magic != OVFS_MAGIC {
+        return Err(invalid("not an OVFS (RankQuantFastscan) file: wrong magic"));
+    }
+    read_version(&mut f, "OVFS")?;
+    let dim = read_u32_le(&mut f, "OVFS", "dim")? as usize;
+    check_dim(dim)?;
+    // FastScan b=2 requires `dim % 4 == 0` (mirrors `RankQuantFastscan::new` /
+    // `RankQuant::new(dim, 2)`: constant composition, exact analytical norm).
+    // `dim % 4 == 0` subsumes the pair-encoding's `dim % 2 == 0`.
+    if !dim.is_multiple_of(4) {
+        return Err(invalid(format!(
+            "OVFS dim {dim} is not a multiple of 4 (b=2 constant composition)"
+        )));
+    }
+    let n_vectors = read_u32_le(&mut f, "OVFS", "n_vectors")? as usize;
+    check_n_vectors(n_vectors)?;
+    let payload_bytes = fastscan_payload_bytes(dim, n_vectors)?;
+    check_payload_bytes(payload_bytes)?;
+    check_payload_matches_file(&mut f, "OVFS", file_len, payload_bytes)?;
+    // The packed FastScan payload is opaque pre-encoded nibbles in the block-32
+    // transpose: any byte value is valid, so there is no per-row invariant to
+    // check beyond the exact payload length validated above.
+    let mut packed_fs = try_alloc_zeroed(payload_bytes)?;
+    f.read_exact(&mut packed_fs)?;
+    Ok((dim, n_vectors, packed_fs))
+}
+
 #[cfg(test)]
 mod tests {
     use super::{
@@ -1477,4 +1571,33 @@ mod tests {
             let _ = std::fs::remove_file(p);
         }
     }
+
+    // OVFS (FastScan) write path: valid round-trip, and fail-loud (io::Error, not
+    // a panic) on invalid `dim`/`n_vectors`/payload — the now-public persistence
+    // API must never abort the caller or silently truncate the header.
+    #[test]
+    fn write_fastscan_validates_and_never_panics() {
+        use super::{load_fastscan, write_fastscan};
+        // dim=8 (multiple of 4), 4 vectors -> ceil(4/32)*(8/2)*32 = 128-byte payload.
+        let (dim, n) = (8usize, 4usize);
+        let payload = vec![0u8; 128];
+        let p = temp_index_path("ovfs_ok");
+        write_fastscan(&p, dim, n, &payload).unwrap();
+        let (ld, ln, lbytes) = load_fastscan(&p).unwrap();
+        assert_eq!((ld, ln), (dim, n));
+        assert_eq!(lbytes, payload, "OVFS round-trip altered the payload");
+        let _ = std::fs::remove_file(&p);
+
+        // dim not a multiple of 4 -> rejected before File::create (no panic, no file).
+        let p2 = temp_index_path("ovfs_baddim");
+        let e = write_fastscan(&p2, 6, n, &payload).unwrap_err();
+        assert_eq!(e.kind(), std::io::ErrorKind::InvalidData);
+        assert!(!p2.exists(), "rejected write must not create a file");
+
+        // packed buffer inconsistent with dim/n_vectors -> rejected, not panic.
+        let p3 = temp_index_path("ovfs_badlen");
+        let e = write_fastscan(&p3, dim, n, &payload[..100]).unwrap_err();
+        assert_eq!(e.kind(), std::io::ErrorKind::InvalidData);
+        assert!(!p3.exists(), "rejected write must not create a file");
+    }
 }