From 51cb3bfba548fd8f67def8576149c7f3e84acc08 Mon Sep 17 00:00:00 2001 From: "zhuliming.zlm" Date: Tue, 14 Jul 2026 16:07:55 +0800 Subject: [PATCH] feat(sindi): support proximity and phrase filter Signed-off-by: xuanji.ch Signed-off-by: zhuliming.zlm Assisted-by: Codex:gpt-5 --- docs/docs/en/src/indexes/sindi.md | 88 + docs/docs/zh/src/indexes/sindi.md | 83 + scripts/gen_sindi_bench.py | 241 +++ src/algorithm/sindi/CMakeLists.txt | 1 + src/algorithm/sindi/proximity_scorer.cpp | 292 +++ src/algorithm/sindi/proximity_scorer.h | 120 ++ src/algorithm/sindi/proximity_scorer_test.cpp | 393 ++++ src/algorithm/sindi/sindi.cpp | 506 ++++- src/algorithm/sindi/sindi.h | 264 ++- src/algorithm/sindi/sindi_parameter.cpp | 73 + src/algorithm/sindi/sindi_parameter.h | 28 + src/algorithm/sindi/sindi_parameter_test.cpp | 106 +- src/algorithm/sindi/sindi_test.cpp | 1887 +++++++++++++++++ src/datacell/sparse_term_datacell.cpp | 151 ++ src/datacell/sparse_term_datacell.h | 36 +- .../sparse_term_datacell_position_test.cpp | 365 ++++ src/inner_string_params.h | 10 + tools/eval/eval_dataset.h | 7 + 18 files changed, 4627 insertions(+), 24 deletions(-) create mode 100644 scripts/gen_sindi_bench.py create mode 100644 src/algorithm/sindi/proximity_scorer.cpp create mode 100644 src/algorithm/sindi/proximity_scorer.h create mode 100644 src/algorithm/sindi/proximity_scorer_test.cpp create mode 100644 src/datacell/sparse_term_datacell_position_test.cpp diff --git a/docs/docs/en/src/indexes/sindi.md b/docs/docs/en/src/indexes/sindi.md index f8444be520..5b8f1c7a52 100644 --- a/docs/docs/en/src/indexes/sindi.md +++ b/docs/docs/en/src/indexes/sindi.md @@ -76,6 +76,8 @@ and `metric_type` **must** be `"ip"`. | `use_quantization` | bool | `false` | Quantize stored term values to cut memory; when enabled, uses 8-bit scalar quantization (SQ8). | | `use_reorder` | bool | `false` | Keep a high-precision flat copy and rescore results (~2× memory). | | `remap_term_ids` | bool | `false` | Remap term IDs before indexing; useful when term IDs are sparse or have large gaps. | +| `store_positions` | bool | `false` | Store per-term positions extracted from each document's original token order. Required to use proximity scoring or the phrase filter at query time. Increases memory roughly proportional to document length. | +| `max_positions_per_term` | int | `64` | Cap on stored positions per term per document (range: 1 – 256). Extra occurrences beyond the cap are dropped. Only relevant when `store_positions` is `true`. | | `avg_doc_term_length` | int | `100` | Hint for memory estimation only. | > **`dim` vs `term_id_limit`.** For the sparse vector `{0:0.1, 2:0.5, 177:0.8}`, @@ -92,6 +94,14 @@ Search-time parameters live under the `sindi` sub-object: | `n_candidate` | int | `0` | Candidate heap size. When `0`, defaults to `SPARSE_AMPLIFICATION_FACTOR · topk` (500×). If set, must satisfy `1 ≤ n_candidate ≤ SPARSE_AMPLIFICATION_FACTOR · topk`. | | `query_prune_ratio` | float | `0.0` | Fraction of lowest-weight query terms skipped (0.0 – 0.9). | | `term_prune_ratio` | float | `0.0` | Fraction of term-list entries skipped (0.0 – 0.9). | +| `proximity_boost` | object | — | Proximity-scoring options. Omit this object to use the defaults below. Requires an index built with `store_positions: true` when enabled. | +| `proximity_boost.weight` | float | `0.0` | Weight of the proximity boost. `0.0` disables proximity scoring; a positive value enables it. | +| `proximity_boost.candidates` | int | `10000` | Number of top cosine candidates re-ranked by proximity within each window. Only these candidates pay the proximity cost. | +| `proximity_boost.all_pairs` | bool | `false` | `true` scores all `C(n,2)` query-term pairs; `false` scores only adjacent pairs `(i, i+1)` in query order. | +| `proximity_boost.ordered` | bool | `false` | When `true`, out-of-order (reversed) term pairs are penalized (their distance is doubled), similar to Lucene's slop cost. | +| `proximity_boost.boost_multiplicative` | bool | `true` | `true` multiplies the base cosine score by the proximity boost; `false` adds the boost. | +| `phrase_terms` | int[] | `[]` | Term ids forming a phrase constraint. When non-empty, a candidate is kept only if these terms occur within `phrase_slop`. Requires `store_positions: true`. | +| `phrase_slop` | int | `0` | Maximum allowed slop (edit distance in positions) for the phrase filter. `0` requires an exact, in-order phrase. | SINDI chooses the heap-insertion strategy automatically from the build-time `doc_prune_ratio` and search-time `query_prune_ratio`. With the current `0.1` @@ -107,6 +117,84 @@ auto result = index->KnnSearch( R"({"sindi": {"n_candidate": 200, "query_prune_ratio": 0.1}})").value(); ``` +## Proximity scoring and phrase filter + +By default SINDI scores documents as a bag of words: two documents with the same +term weights score identically regardless of where the terms appear. When term +*position* matters — phrase-like queries, "these words should appear together" — +SINDI can optionally boost documents whose query terms are close, or hard-filter +documents that don't contain a phrase. + +Both features require positions to be stored at build time and the original +tokenized document to be supplied per vector. + +1. **Build with `store_positions: true`.** +2. **Supply the original token order** on each `SparseVector` via + `token_sequence_` / `token_seq_len_`. Unlike `ids_` (deduplicated and sorted), + `token_sequence_` preserves the raw tokenization order and duplicates, which + is what positions are extracted from. + +```cpp +// Build side: enable position storage. +std::string params = R"({ + "dtype": "sparse", + "metric_type": "ip", + "dim": 1024, + "index_param": { + "term_id_limit": 30000, + "window_size": 50000, + "store_positions": true, + "max_positions_per_term": 64 + } +})"; + +// Per document: keep the original tokenized term ids. +vsag::SparseVector doc; +doc.len_ = n_nonzero; // deduplicated ids_/vals_ as usual +doc.ids_ = ids; +doc.vals_ = vals; +doc.token_seq_len_ = seq_len; // original token order, with duplicates +doc.token_sequence_ = token_ids; +``` + +**Proximity boost.** Set `proximity_boost.weight > 0` to re-rank the top +`proximity_boost.candidates` within each window (by cosine) using a pairwise +`1/(min_dist + 1)` boost over query terms: + +```cpp +auto result = index->KnnSearch( + query, topk, + R"({"sindi": { + "n_candidate": 200, + "proximity_boost": { + "weight": 0.3, + "candidates": 10000, + "all_pairs": false, + "ordered": false, + "boost_multiplicative": true + } + }})").value(); +``` + +**Phrase filter.** Set `phrase_terms` (and optionally `phrase_slop`) to keep +only candidates where those terms occur within the given slop. Slop uses +Lucene's normalized-window semantics, so term order is encoded in the offsets +(reversals cost extra slop); `phrase_slop: 0` requires an exact in-order phrase: + +```cpp +auto result = index->KnnSearch( + query, topk, + R"({"sindi": { + "n_candidate": 200, + "phrase_terms": [12, 34, 56], + "phrase_slop": 1 + }})").value(); +``` + +Proximity and phrase can be combined in the same query. Per-candidate cost is +`O(T² × P)` (T = present query terms, P = positions per term), paid only by the +re-ranked candidates. + ## When to use SINDI - Sparse retrieval with BM25, SPLADE, uniCOIL, or similar learned-sparse encoders. diff --git a/docs/docs/zh/src/indexes/sindi.md b/docs/docs/zh/src/indexes/sindi.md index 823198b5e4..e596788c83 100644 --- a/docs/docs/zh/src/indexes/sindi.md +++ b/docs/docs/zh/src/indexes/sindi.md @@ -72,6 +72,8 @@ auto result = index->KnnSearch( | `use_quantization` | bool | `false` | 是否量化词项权重以降低内存；开启后使用 8-bit 标量量化（SQ8） | | `use_reorder` | bool | `false` | 是否保留一份高精度扁平副本用于精排（内存约翻倍） | | `remap_term_ids` | bool | `false` | 是否在建索引前重映射词项 ID，适用于词项 ID 很稀疏或存在大量空洞的词表 | +| `store_positions` | bool | `false` | 是否存储每个词项在文档原始 token 顺序中的位置。启用邻近度打分或短语过滤时必须开启；内存开销大致与文档长度成正比 | +| `max_positions_per_term` | int | `64` | 每篇文档中每个词项最多存储的位置数量（取值范围 1 – 256），超出的出现将被丢弃。仅在 `store_positions` 为 `true` 时生效 | | `avg_doc_term_length` | int | `100` | 仅用于内存估算 | > **`dim` 与 `term_id_limit` 的区别。** 对于稀疏向量 `{0:0.1, 2:0.5, 177:0.8}`， @@ -87,6 +89,14 @@ auto result = index->KnnSearch( | `n_candidate` | int | `0` | 候选堆大小。为 `0` 时自动取 `SPARSE_AMPLIFICATION_FACTOR · topk`（500 倍）；若显式设置，须满足 `1 ≤ n_candidate ≤ SPARSE_AMPLIFICATION_FACTOR · topk` | | `query_prune_ratio` | float | `0.0` | 查询时丢弃权重最低查询项的比例（0.0 – 0.9） | | `term_prune_ratio` | float | `0.0` | 查询时丢弃倒排表中低权项的比例（0.0 – 0.9） | +| `proximity_boost` | object | — | 邻近度打分选项；省略时使用下列默认值。启用时要求索引构建阶段设置 `store_positions: true` | +| `proximity_boost.weight` | float | `0.0` | 邻近度加权的权重。`0.0` 表示关闭邻近度打分，正值开启 | +| `proximity_boost.candidates` | int | `10000` | 每个 window 内按余弦得分参与邻近度重排的候选数量，只有这些候选会承担邻近度计算开销 | +| `proximity_boost.all_pairs` | bool | `false` | `true` 对所有 `C(n,2)` 查询词项对打分；`false` 仅对查询顺序中相邻的词项对 `(i, i+1)` 打分 | +| `proximity_boost.ordered` | bool | `false` | 为 `true` 时对逆序（reversed）词项对施加惩罚（距离翻倍），类似 Lucene 的 slop 代价 | +| `proximity_boost.boost_multiplicative` | bool | `true` | `true` 将邻近度加权与基础余弦得分相乘；`false` 表示相加 | +| `phrase_terms` | int[] | `[]` | 构成短语约束的词项 ID 列表。非空时，只有这些词项在 `phrase_slop` 内共现的候选才会保留。要求 `store_positions: true` | +| `phrase_slop` | int | `0` | 短语过滤允许的最大 slop（位置上的编辑距离）。`0` 表示要求精确、按序的短语 | SINDI 会根据构建阶段的 `doc_prune_ratio` 与检索阶段的 `query_prune_ratio` 自动选择堆插入策略。按当前 `0.1` 阈值，当两个比例都 `<= 0.1` 时，SINDI 使用 @@ -99,6 +109,79 @@ auto result = index->KnnSearch( R"({"sindi": {"n_candidate": 200, "query_prune_ratio": 0.1}})").value(); ``` +## 邻近度打分与短语过滤 + +默认情况下 SINDI 以词袋（bag of words）方式打分：只要词项权重相同，两篇文档得分 +相同，与词项出现的位置无关。当词项 *位置* 很重要时——例如短语类查询、“这些词应当 +挨在一起”——SINDI 可以选择性地对查询词项相互靠近的文档加权，或直接过滤掉不包含某个 +短语的文档。 + +这两项能力都要求构建时存储位置，并且逐条向量提供原始的分词序列。 + +1. **构建时设置 `store_positions: true`。** +2. **在每条 `SparseVector` 上提供原始 token 顺序**，通过 `token_sequence_` / + `token_seq_len_` 传入。与 `ids_`（去重且排序）不同，`token_sequence_` 保留原始 + 分词顺序和重复项，位置正是从中提取的。 + +```cpp +// 构建侧：开启位置存储。 +std::string params = R"({ + "dtype": "sparse", + "metric_type": "ip", + "dim": 1024, + "index_param": { + "term_id_limit": 30000, + "window_size": 50000, + "store_positions": true, + "max_positions_per_term": 64 + } +})"; + +// 逐篇文档：保留原始分词后的词项 ID。 +vsag::SparseVector doc; +doc.len_ = n_nonzero; // 与平时一样的去重 ids_/vals_ +doc.ids_ = ids; +doc.vals_ = vals; +doc.token_seq_len_ = seq_len; // 原始 token 顺序，含重复 +doc.token_sequence_ = token_ids; +``` + +**邻近度加权。** 设置 `proximity_boost.weight > 0`，即可对每个 window 内按余弦得分 +排序的前 `proximity_boost.candidates` 个候选，使用基于查询词项对的 +`1/(min_dist + 1)` 加权进行重排： + +```cpp +auto result = index->KnnSearch( + query, topk, + R"({"sindi": { + "n_candidate": 200, + "proximity_boost": { + "weight": 0.3, + "candidates": 10000, + "all_pairs": false, + "ordered": false, + "boost_multiplicative": true + } + }})").value(); +``` + +**短语过滤。** 设置 `phrase_terms`（可选 `phrase_slop`），只保留这些词项在给定 +slop 内共现的候选。slop 采用 Lucene 的归一化窗口语义，词项顺序被编码进偏移量中 +（逆序会消耗额外 slop）；`phrase_slop: 0` 要求精确、按序的短语： + +```cpp +auto result = index->KnnSearch( + query, topk, + R"({"sindi": { + "n_candidate": 200, + "phrase_terms": [12, 34, 56], + "phrase_slop": 1 + }})").value(); +``` + +邻近度与短语过滤可在同一次查询中组合使用。每个候选的开销为 `O(T² × P)`（T 为命中的 +查询词项数，P 为每词项位置数），且只有参与重排的候选才会付出该开销。 + ## 何时选择 SINDI - 使用 BM25、SPLADE、uniCOIL 等学习稀疏编码器的稀疏检索场景。 diff --git a/scripts/gen_sindi_bench.py b/scripts/gen_sindi_bench.py new file mode 100644 index 0000000000..3ba7065d87 --- /dev/null +++ b/scripts/gen_sindi_bench.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +""" +Generate SINDI benchmark HDF5 dataset with proximity-aware ground truth. + +Usage: + python gen_sindi_bench.py --output sindi_bench.hdf5 \ + --num_base 10000 --num_query 100 --doc_len 512 \ + --vocab_size 5000 --max_terms 200 \ + --proximity_weight 0.3 --topk 10 + +Output HDF5 structure: + /train - serialized sparse vectors (binary blob) + /test - serialized query sparse vectors (binary blob) + /neighbors - ground truth neighbor IDs (int64, Q x K) + /distances - ground truth distances (float32, Q x K) + attrs["distance"] = "ip" +""" + +import argparse +import struct + +import h5py +import numpy as np +from collections import defaultdict + + +def generate_sparse_doc(rng, vocab_size, max_terms, doc_len): + """Generate a random sparse document with token sequence.""" + # Token sequence: random tokens from vocab + token_seq = rng.integers(0, vocab_size, size=doc_len, dtype=np.uint32) + + # Sparse vector: unique terms with random weights + unique_terms = np.unique(token_seq) + if len(unique_terms) > max_terms: + # Keep top max_terms by frequency + counts = np.bincount(token_seq, minlength=vocab_size) + top_terms = np.argsort(counts)[::-1][:max_terms] + unique_terms = np.sort(top_terms[counts[top_terms] > 0]) + + weights = rng.uniform(0.1, 1.0, size=len(unique_terms)).astype(np.float32) + return unique_terms.astype(np.uint32), weights, token_seq + + +def serialize_sparse_vectors(vectors): + """Serialize sparse vectors to binary blob for HDF5. + + Each vector: [len(uint32), ids(uint32 x len), vals(float32 x len)] + """ + parts = [] + for ids, vals, _token_seq in vectors: + parts.append(struct.pack(' 0 and d_token_seq is not None: + boost = compute_pairwise_proximity(q_ids, d_ids, d_token_seq, proximity_ordered) + if multiplicative: + ip = ip * (1 + proximity_weight * boost) + else: + ip = ip + proximity_weight * boost + + return 1.0 - ip + + +def main(): + parser = argparse.ArgumentParser(description='Generate SINDI benchmark HDF5') + parser.add_argument('--output', default='sindi_bench.hdf5') + parser.add_argument('--num_base', type=int, default=10000) + parser.add_argument('--num_query', type=int, default=100) + parser.add_argument('--doc_len', type=int, default=512) + parser.add_argument('--vocab_size', type=int, default=5000) + parser.add_argument('--max_terms', type=int, default=200) + parser.add_argument('--proximity_weight', type=float, default=0.0, + help='0.0 for pure IP ground truth, >0 for proximity-boosted GT') + parser.add_argument('--proximity_ordered', action='store_true') + parser.add_argument('--multiplicative', action='store_true', default=True) + parser.add_argument('--topk', type=int, default=10) + parser.add_argument('--seed', type=int, default=42) + args = parser.parse_args() + + rng = np.random.default_rng(args.seed) + print(f"Generating {args.num_base} docs, {args.num_query} queries, " + f"vocab={args.vocab_size}, doc_len={args.doc_len}") + + # Generate base docs + base_docs = [] + for i in range(args.num_base): + ids, vals, token_seq = generate_sparse_doc( + rng, args.vocab_size, args.max_terms, args.doc_len) + base_docs.append((ids, vals, token_seq)) + if (i + 1) % 1000 == 0: + print(f" Generated {i + 1}/{args.num_base} docs") + + # Generate queries (sample from base docs, use their terms as query) + query_indices = rng.choice(args.num_base, size=args.num_query, replace=False) + queries = [] + for idx in query_indices: + ids, vals, _ = base_docs[idx] + # Use a subset of terms as query + n_query_terms = min(len(ids), 10) + sel = rng.choice(len(ids), size=n_query_terms, replace=False) + sel.sort() + q_ids = ids[sel] + q_vals = vals[sel] + queries.append((q_ids, q_vals, None)) + + # Compute brute-force ground truth + print(f"Computing ground truth (proximity_weight={args.proximity_weight})...") + neighbors = np.zeros((args.num_query, args.topk), dtype=np.int64) + distances = np.zeros((args.num_query, args.topk), dtype=np.float32) + + for qi in range(args.num_query): + q_ids, q_vals, _ = queries[qi] + dists = [] + for di in range(args.num_base): + d_ids, d_vals, d_token_seq = base_docs[di] + d = compute_distance( + q_ids, q_vals, d_ids, d_vals, + d_token_seq if args.proximity_weight > 0 else None, + args.proximity_weight, args.proximity_ordered, args.multiplicative) + dists.append((d, di)) + dists.sort() + for k in range(args.topk): + distances[qi, k] = dists[k][0] + neighbors[qi, k] = dists[k][1] + if (qi + 1) % 10 == 0: + print(f" Query {qi + 1}/{args.num_query}") + + # Serialize and write HDF5 + print(f"Writing {args.output}...") + train_blob = serialize_sparse_vectors(base_docs) + test_blob = serialize_sparse_vectors( + [(q[0], q[1], np.array([], dtype=np.uint32)) for q in queries]) + train_seq_blob, train_seq_offsets = serialize_token_sequences(base_docs) + + with h5py.File(args.output, 'w') as f: + f.create_dataset('train', data=train_blob) + f.create_dataset('test', data=test_blob) + f.create_dataset('train_token_sequences', data=train_seq_blob) + f.create_dataset('train_token_sequences_offsets', data=train_seq_offsets) + f.create_dataset('neighbors', data=neighbors) + f.create_dataset('distances', data=distances) + f.attrs['distance'] = 'ip' + # Set type attribute for sparse vectors (required by eval_dataset.cpp) + f.attrs['type'] = 'sparse' + + print(f"Done. {args.output}: {args.num_base} docs, {args.num_query} queries, " + f"top-{args.topk} GT") + + +if __name__ == '__main__': + main() diff --git a/src/algorithm/sindi/CMakeLists.txt b/src/algorithm/sindi/CMakeLists.txt index 6432269ce9..7a983f8b19 100644 --- a/src/algorithm/sindi/CMakeLists.txt +++ b/src/algorithm/sindi/CMakeLists.txt @@ -18,6 +18,7 @@ set (SINDI_SRCS sindi.cpp sindi_parameter.cpp term_id_mapper.cpp + proximity_scorer.cpp ) add_library (sindi OBJECT ${SINDI_SRCS}) diff --git a/src/algorithm/sindi/proximity_scorer.cpp b/src/algorithm/sindi/proximity_scorer.cpp new file mode 100644 index 0000000000..d5e860cd80 --- /dev/null +++ b/src/algorithm/sindi/proximity_scorer.cpp @@ -0,0 +1,292 @@ + +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "proximity_scorer.h" + +#include +#include +#include + +namespace vsag { + +namespace { + +// Find the minimum distance between two sorted position lists. +// For unordered mode: returns min |a - b| over all (a, b) pairs. +// For ordered mode: returns min distance considering forward/reverse penalty. +// Forward (a < b): dist = b - a +// Reverse (a > b): dist = (a - b) * 2 +uint32_t +min_distance_between_lists(const PosSpan& list_a, const PosSpan& list_b, bool ordered) { + uint32_t min_dist = std::numeric_limits::max(); + + // Both lists are expected to be in insertion order (ascending for positions + // from a single document scan). Use two-pointer merge for efficiency. + if (!ordered) { + // Unordered: classic sorted merge to find min |a - b| + uint64_t i = 0; + uint64_t j = 0; + while (i < list_a.size && j < list_b.size) { + uint32_t a = list_a[i]; + uint32_t b = list_b[j]; + uint32_t dist = (a > b) ? (a - b) : (b - a); + if (dist < min_dist) { + min_dist = dist; + } + if (min_dist == 0) { + break; + } + if (a < b) { + ++i; + } else { + ++j; + } + } + } else { + // Ordered: two-pointer merge to find best forward or penalized reverse distance. + uint64_t i = 0; + uint64_t j = 0; + while (i < list_a.size && j < list_b.size) { + uint32_t a = list_a[i]; + uint32_t b = list_b[j]; + uint32_t dist; + if (a <= b) { + dist = b - a; + ++i; + } else { + dist = (a - b) * 2; + ++j; + } + if (dist < min_dist) { + min_dist = dist; + } + } + } + + return min_dist; +} + +} // namespace + +float +all_pairwise_proximity(const std::vector& position_lists, bool ordered) { + float boost = 0.0f; + uint64_t n = position_lists.size(); + + for (uint64_t i = 0; i < n; ++i) { + if (position_lists[i].empty()) { + continue; + } + for (uint64_t j = i + 1; j < n; ++j) { + if (position_lists[j].empty()) { + continue; + } + uint32_t dist = + min_distance_between_lists(position_lists[i], position_lists[j], ordered); + if (dist < std::numeric_limits::max()) { + boost += 1.0f / static_cast(dist + 1); + } + } + } + + return boost; +} + +float +adjacent_pairwise_proximity(const std::vector& position_lists, bool ordered) { + float boost = 0.0f; + uint64_t n = position_lists.size(); + + // Only score adjacent pairs (i, i+1) in query order → n-1 pairs. + for (uint64_t i = 0; i + 1 < n; ++i) { + if (position_lists[i].empty() || position_lists[i + 1].empty()) { + continue; + } + uint32_t dist = + min_distance_between_lists(position_lists[i], position_lists[i + 1], ordered); + if (dist < std::numeric_limits::max()) { + boost += 1.0f / static_cast(dist + 1); + } + } + + return boost; +} + +namespace { + +// File-private helper. It lives in an anonymous namespace so it has *internal +// linkage* (visible only inside this translation unit, the modern replacement +// for a file-level `static` function); it is the shared core of the two +// sloppy_phrase_match overloads below and is not meant to be called elsewhere. +// +// --- What "sloppy phrase" means --- +// For a query phrase like "A B C", a document still matches even if the terms +// are not perfectly consecutive, as long as their layout is within a bounded +// "slop" of the ideal. This mirrors Lucene's SloppyPhraseMatcher. +// +// --- The normalization trick --- +// Query term i has an ideal offset i (A=0, B=1, C=2). For a real position `pos` +// of term i we store norm = pos - i. If the terms are perfectly consecutive, +// every term's norm collapses to the SAME value, so the spread +// (max_norm - min_norm) is 0; the more the layout deviates, the larger the +// spread. A reversed pair produces a smaller/negative norm, which widens the +// spread and therefore "costs" extra slop -- exactly Lucene's behavior. +// +// --- The sweep --- +// `all_norms` holds one {norm, term_idx} entry per position of every term (the +// caller builds it). We sort it by norm, then slide a window [left, right] over +// the sorted entries, looking for the smallest window that contains at least +// one entry from EVERY term (terms_covered == n). `term_count[t]` tracks how +// many entries of term t are currently inside the window; it must be sized to n +// and is used purely as counting scratch. As soon as a covering window has +// spread <= slop we return true. +// +// --- Worked example --- +// query "A B C" (ideal offsets 0,1,2); document has A@0, B@2, C@4. +// norms: A: 0-0=0, B: 2-1=1, C: 4-2=2 +// sorted: [ (norm=0,A), (norm=1,B), (norm=2,C) ] +// the first window covering {A,B,C} is [0..2], spread = 2 - 0 = 2. +// => matches when slop >= 2, fails when slop < 2. +// +// `all_norms` is sorted in place. Returns true iff some covering window has +// (max_norm - min_norm) <= slop. +bool +sloppy_phrase_sweep(std::vector& all_norms, + std::vector& term_count, + uint64_t n, + uint32_t slop) { + std::sort(all_norms.begin(), all_norms.end(), [](const NormEntry& a, const NormEntry& b) { + return a.norm < b.norm; + }); + + uint64_t terms_covered = 0; + uint64_t left = 0; + for (uint64_t right = 0; right < all_norms.size(); ++right) { + auto idx = all_norms[right].term_idx; + if (term_count[idx] == 0) { + terms_covered++; + } + term_count[idx]++; + + while (terms_covered == n) { + int32_t distance = all_norms[right].norm - all_norms[left].norm; + if (distance <= static_cast(slop)) { + return true; + } + auto left_idx = all_norms[left].term_idx; + term_count[left_idx]--; + if (term_count[left_idx] == 0) { + terms_covered--; + } + left++; + } + } + return false; +} + +} // namespace + +bool +sloppy_phrase_match(const std::vector>& phrase_term_positions, + uint32_t slop) { + uint64_t n = phrase_term_positions.size(); + if (n <= 1) { + return true; + } + + // All terms must be present. + for (uint64_t i = 0; i < n; ++i) { + if (phrase_term_positions[i].empty()) { + return false; + } + } + + // Normalize each position by its query offset (term index): norm = pos - i. + // Reversals produce smaller/negative norms, widening the window (i.e. they + // cost extra slop) exactly as Lucene's SloppyPhraseMatcher does. + std::vector all_norms; + for (uint64_t i = 0; i < n; ++i) { + for (auto pos : phrase_term_positions[i]) { + all_norms.push_back({static_cast(pos) - static_cast(i), i}); + } + } + std::vector term_count(n, 0); + return sloppy_phrase_sweep(all_norms, term_count, n, slop); +} + +bool +sloppy_phrase_match(const std::vector& phrase_term_positions, + uint32_t slop, + std::vector& norm_buffer, + std::vector& count_buffer) { + uint64_t n = phrase_term_positions.size(); + if (n <= 1) { + return true; + } + + for (uint64_t i = 0; i < n; ++i) { + if (phrase_term_positions[i].empty()) { + return false; + } + } + + norm_buffer.clear(); + for (uint64_t i = 0; i < n; ++i) { + const auto& span = phrase_term_positions[i]; + for (uint32_t k = 0; k < span.size; ++k) { + norm_buffer.push_back({static_cast(span[k]) - static_cast(i), i}); + } + } + count_buffer.assign(n, 0); + return sloppy_phrase_sweep(norm_buffer, count_buffer, n, slop); +} + +void +extract_positions_from_token_sequence(const uint32_t* token_sequence, + uint32_t seq_len, + const uint32_t* ids, + uint32_t ids_len, + uint32_t max_positions_per_term, + std::vector>& out_positions) { + out_positions.clear(); + out_positions.resize(ids_len); + + if (token_sequence == nullptr || seq_len == 0 || ids == nullptr || ids_len == 0) { + return; + } + + // Build a map from term_id → index in ids[] for O(1) lookup + std::unordered_map> term_to_indices; + for (uint32_t i = 0; i < ids_len; ++i) { + term_to_indices[ids[i]].push_back(i); + } + + // Scan token sequence and collect positions. Positions are stored as uint16_t. + constexpr uint32_t kMaxStorablePosition = + static_cast(std::numeric_limits::max()) + 1; + uint32_t limit = std::min(seq_len, kMaxStorablePosition); + for (uint32_t pos = 0; pos < limit; ++pos) { + auto it = term_to_indices.find(token_sequence[pos]); + if (it != term_to_indices.end()) { + for (uint32_t idx : it->second) { + if (out_positions[idx].size() < max_positions_per_term) { + out_positions[idx].push_back(static_cast(pos)); + } + } + } + } +} + +} // namespace vsag diff --git a/src/algorithm/sindi/proximity_scorer.h b/src/algorithm/sindi/proximity_scorer.h new file mode 100644 index 0000000000..38f58be981 --- /dev/null +++ b/src/algorithm/sindi/proximity_scorer.h @@ -0,0 +1,120 @@ + +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +namespace vsag { + +// Non-owning view over a contiguous, immutable position list. Used by the +// proximity hot path to avoid per-candidate vector copies; the underlying data +// lives in SparseTermDataCell::term_pos_pool_ and is read-only during a query. +struct PosSpan { + const uint16_t* data{nullptr}; + uint32_t size{0}; + + bool + empty() const { + return size == 0; + } + const uint16_t* + begin() const { + return data; + } + const uint16_t* + end() const { + return data + size; + } + uint16_t + operator[](uint32_t i) const { + return data[i]; + } +}; + +// Compute pairwise proximity boost over all C(n,2) query-term pairs. +// +// For each pair of terms (i, j) where i < j, finds the minimum distance between +// their position lists and accumulates 1/(min_dist + 1). +// +// When ordered=true, the distance for reversed pairs (pos_i > pos_j) is doubled +// as a penalty (similar to Lucene's slop cost for reversals). +// +// Returns 0.0 if fewer than 2 non-empty position lists are provided. +float +all_pairwise_proximity(const std::vector& position_lists, bool ordered); + +// Simplified proximity boost that only scores adjacent query-term pairs. +// +// Unlike all_pairwise_proximity (which scores all C(n,2) pairs), this scores +// only the n-1 pairs (i, i+1), where adjacency follows the order of +// position_lists (i.e. the query's raw_query_.ids_ order). For query ABC it +// scores (A,B) and (B,C) but not (A,C). +// +// The ordered flag and per-pair distance/boost semantics match +// all_pairwise_proximity. Empty position lists contribute nothing. +float +adjacent_pairwise_proximity(const std::vector& position_lists, bool ordered); + +// Scratch entry for sloppy_phrase_match's normalized-window sweep. Exposed so +// hot-path callers can hoist the scratch buffers out of their per-doc loop and +// reuse them across calls (see the PosSpan overload below). +struct NormEntry { + int32_t norm; + uint64_t term_idx; +}; + +// Phrase constraint using Lucene SloppyPhraseMatcher's normalized-window slop. +// +// Each term's positions are normalized by their query offset (the term's index +// in phrase_term_positions): norm = doc_pos - term_idx. The match distance is +// max(norm) - min(norm) over a window covering one position per term; the +// constraint passes if any such window has distance <= slop. This encodes order +// into the offsets (reversals cost extra slop) so there is no ordered/unordered +// distinction. Returns true as soon as one satisfying window is found. +// +// All terms must be present (non-empty). Returns true for 0 or 1 terms. +bool +sloppy_phrase_match(const std::vector>& phrase_term_positions, uint32_t slop); + +// Zero-copy overload over PosSpan position lists. Identical semantics to the +// vector> version, but avoids per-doc position copies and +// reuses caller-provided scratch buffers (norm_buffer / count_buffer) instead +// of allocating them on every call. The scratch contents are overwritten each +// call; callers only need to keep the buffers alive for reuse. +bool +sloppy_phrase_match(const std::vector& phrase_term_positions, + uint32_t slop, + std::vector& norm_buffer, + std::vector& count_buffer); + +// Extract per-term positions from a raw token sequence. +// +// For each term in ids[0..ids_len), scans token_sequence to find all positions +// where that term appears. Caps each term's positions at max_positions_per_term. +// Only terms present in ids[] are extracted; other tokens are ignored. +// +// out_positions is resized to ids_len, with out_positions[i] containing the +// positions for ids[i]. +void +extract_positions_from_token_sequence(const uint32_t* token_sequence, + uint32_t seq_len, + const uint32_t* ids, + uint32_t ids_len, + uint32_t max_positions_per_term, + std::vector>& out_positions); + +} // namespace vsag diff --git a/src/algorithm/sindi/proximity_scorer_test.cpp b/src/algorithm/sindi/proximity_scorer_test.cpp new file mode 100644 index 0000000000..a2efc4aa52 --- /dev/null +++ b/src/algorithm/sindi/proximity_scorer_test.cpp @@ -0,0 +1,393 @@ + +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "proximity_scorer.h" + +#include +#include +#include + +#include "unittest.h" + +#define REQUIRE_APPROX(a, b) REQUIRE(std::abs((a) - (b)) < 1e-6f) + +using namespace vsag; + +static std::vector +to_spans(const std::vector>& lists) { + std::vector spans; + spans.reserve(lists.size()); + for (const auto& list : lists) { + spans.push_back(PosSpan{list.data(), static_cast(list.size())}); + } + return spans; +} + +TEST_CASE("ProximityScorer Empty Input", "[ut][ProximityScorer]") { + // No position lists at all + std::vector> empty; + float boost = all_pairwise_proximity(to_spans(empty), false); + REQUIRE(boost == 0.0f); +} + +TEST_CASE("ProximityScorer Single Term", "[ut][ProximityScorer]") { + // Only one term present — no pairs, boost should be 0 + std::vector> single = {{5, 10, 20}}; + float boost = all_pairwise_proximity(to_spans(single), false); + REQUIRE(boost == 0.0f); +} + +TEST_CASE("ProximityScorer Two Terms Adjacent", "[ut][ProximityScorer]") { + // term A at pos 0, term B at pos 1 → dist=1 → 1/(1+1) = 0.5 + std::vector> positions = {{0}, {1}}; + float boost = all_pairwise_proximity(to_spans(positions), false); + REQUIRE_APPROX(boost, 0.5f); +} + +TEST_CASE("ProximityScorer Two Terms Same Position", "[ut][ProximityScorer]") { + // term A at pos 5, term B at pos 5 → dist=0 → 1/(0+1) = 1.0 + std::vector> positions = {{5}, {5}}; + float boost = all_pairwise_proximity(to_spans(positions), false); + REQUIRE_APPROX(boost, 1.0f); +} + +TEST_CASE("ProximityScorer Two Terms Far Apart", "[ut][ProximityScorer]") { + // term A at pos 0, term B at pos 100 → dist=100 → 1/101 ≈ 0.0099 + std::vector> positions = {{0}, {100}}; + float boost = all_pairwise_proximity(to_spans(positions), false); + REQUIRE_APPROX(boost, 1.0f / 101.0f); +} + +TEST_CASE("ProximityScorer Two Terms Multiple Positions", "[ut][ProximityScorer]") { + // term A at [0, 50, 200], term B at [3, 48, 300] + // min_dist = min(|0-3|, |0-48|, |0-300|, |50-3|, |50-48|, |50-300|, |200-3|, ...) = |50-48| = 2 + // But the algorithm should find min_dist using sorted merge: min over all pairs = 2 + // boost = 1/(2+1) = 0.333... + std::vector> positions = {{0, 50, 200}, {3, 48, 300}}; + float boost = all_pairwise_proximity(to_spans(positions), false); + REQUIRE_APPROX(boost, 1.0f / 3.0f); +} + +TEST_CASE("ProximityScorer Three Terms Pairwise", "[ut][ProximityScorer]") { + // term A at pos 0, term B at pos 1, term C at pos 200 + // pairs: (A,B) dist=1 → 0.5, (A,C) dist=200 → 1/201, (B,C) dist=199 → 1/200 + // total ≈ 0.5 + 0.00498 + 0.005 ≈ 0.51 + std::vector> positions = {{0}, {1}, {200}}; + float boost = all_pairwise_proximity(to_spans(positions), false); + float expected = 1.0f / 2.0f + 1.0f / 201.0f + 1.0f / 200.0f; + REQUIRE_APPROX(boost, expected); +} + +TEST_CASE("ProximityScorer Ordered Forward", "[ut][ProximityScorer]") { + // ordered=true, term A at pos 0, term B at pos 5 + // A before B in query order, pos_A < pos_B → forward → dist = 5 + // boost = 1/(5+1) = 1/6 + std::vector> positions = {{0}, {5}}; + float boost = all_pairwise_proximity(to_spans(positions), true); + REQUIRE_APPROX(boost, 1.0f / 6.0f); +} + +TEST_CASE("ProximityScorer Ordered Reverse Penalty", "[ut][ProximityScorer]") { + // ordered=true, term A at pos 10, term B at pos 5 + // A before B in query order, but pos_A > pos_B → reverse → dist = (10-5)*2 = 10 + // boost = 1/(10+1) = 1/11 + std::vector> positions = {{10}, {5}}; + float boost = all_pairwise_proximity(to_spans(positions), true); + REQUIRE_APPROX(boost, 1.0f / 11.0f); +} + +TEST_CASE("ProximityScorer Ordered vs Unordered", "[ut][ProximityScorer]") { + // Same positions, ordered should give lower boost due to penalty + std::vector> positions = {{10}, {5}}; + float boost_unordered = all_pairwise_proximity(to_spans(positions), false); + float boost_ordered = all_pairwise_proximity(to_spans(positions), true); + // Unordered: dist=5, boost=1/6 + // Ordered: dist=10 (reverse penalty), boost=1/11 + REQUIRE(boost_unordered > boost_ordered); + REQUIRE_APPROX(boost_unordered, 1.0f / 6.0f); + REQUIRE_APPROX(boost_ordered, 1.0f / 11.0f); +} + +TEST_CASE("ProximityScorer Multiple Positions Min Distance", "[ut][ProximityScorer]") { + // term A at [10, 100], term B at [12, 95] + // Unordered min_dist: min(|10-12|, |10-95|, |100-12|, |100-95|) = min(2,85,88,5) = 2 + // boost = 1/3 + std::vector> positions = {{10, 100}, {12, 95}}; + float boost = all_pairwise_proximity(to_spans(positions), false); + REQUIRE_APPROX(boost, 1.0f / 3.0f); +} + +TEST_CASE("ProximityScorer Ordered Multiple Positions Best Forward", "[ut][ProximityScorer]") { + // ordered=true, term A at [10, 100], term B at [12, 95] + // Forward pairs (pos_A < pos_B): (10,12) dist=2, (10,95) dist=85, (100, -) none forward + // Reverse pairs (pos_A > pos_B): (100,12) dist=88*2=176, (100,95) dist=5*2=10 + // Min dist overall: 2 (forward pair 10→12) + // boost = 1/3 + std::vector> positions = {{10, 100}, {12, 95}}; + float boost = all_pairwise_proximity(to_spans(positions), true); + REQUIRE_APPROX(boost, 1.0f / 3.0f); +} + +TEST_CASE("ProximityScorer Empty Position List Skipped", "[ut][ProximityScorer]") { + // term A has positions, term B has empty (term not present in doc) + // This pair contributes nothing + std::vector> positions = {{5, 10}, {}}; + float boost = all_pairwise_proximity(to_spans(positions), false); + REQUIRE(boost == 0.0f); +} + +// ===== adjacent_pairwise_proximity (adjacent-only) tests ===== + +TEST_CASE("CalcProximity Adjacent Only Three Terms", "[ut][ProximityScorer]") { + // A at 0, B at 1, C at 10. Adjacent pairs: (A,B) dist=1 → 0.5, + // (B,C) dist=9 → 0.1. (A,C) is NOT scored. Total = 0.6. + // all_pairwise_proximity would also add (A,C) dist=10 → 1/11 ≈ 0.0909. + std::vector> positions = {{0}, {1}, {10}}; + float adj = adjacent_pairwise_proximity(to_spans(positions), false); + REQUIRE_APPROX(adj, 0.5f + 0.1f); + float all = all_pairwise_proximity(to_spans(positions), false); + REQUIRE(all > adj); // all-pairs includes the extra (A,C) contribution +} + +TEST_CASE("CalcProximity Adjacent Only Empty Term Skipped", "[ut][ProximityScorer]") { + // B empty → both (A,B) and (B,C) skipped → boost 0. + std::vector> positions = {{0}, {}, {2}}; + float boost = adjacent_pairwise_proximity(to_spans(positions), false); + REQUIRE(boost == 0.0f); +} + +TEST_CASE("CalcProximity Adjacent Only Single Term", "[ut][ProximityScorer]") { + std::vector> single = {{5, 10}}; + float boost = adjacent_pairwise_proximity(to_spans(single), false); + REQUIRE(boost == 0.0f); +} + +TEST_CASE("CalcProximity Adjacent Only Ordered Reverse Penalty", "[ut][ProximityScorer]") { + // A at 5, B at 2. Ordered reverse: dist = (5-2)*2 = 6 → 1/7. + std::vector> positions = {{5}, {2}}; + float boost = adjacent_pairwise_proximity(to_spans(positions), true); + REQUIRE_APPROX(boost, 1.0f / 7.0f); +} + +// ===== sloppy_phrase_match (Lucene normalized slop) tests ===== + +TEST_CASE("SloppyPhrase Forward Distance Two", "[ut][ProximityScorer][PhraseFilter]") { + // query offsets [0,1,2], doc pos [0,2,4] → norms [0,1,2] → distance=2. + // (Aligns with lucene_sloppy_freq_algo.md example 1.) + std::vector> positions = {{0}, {2}, {4}}; + REQUIRE(sloppy_phrase_match(positions, 2) == true); + REQUIRE(sloppy_phrase_match(positions, 1) == false); +} + +TEST_CASE("SloppyPhrase Reverse Costs More Slop", "[ut][ProximityScorer][PhraseFilter]") { + // query offsets [0,1,2], doc pos [4,2,0] → norms [4,1,-2] → distance=6. + // (Aligns with lucene_sloppy_freq_algo.md example 2.) + std::vector> positions = {{4}, {2}, {0}}; + REQUIRE(sloppy_phrase_match(positions, 6) == true); + REQUIRE(sloppy_phrase_match(positions, 5) == false); +} + +TEST_CASE("SloppyPhrase Exact Adjacent", "[ut][ProximityScorer][PhraseFilter]") { + // doc pos [0,1] → norms [0,0] → distance=0 → slop=0 passes. + std::vector> positions = {{0}, {1}}; + REQUIRE(sloppy_phrase_match(positions, 0) == true); +} + +TEST_CASE("SloppyPhrase Missing Term", "[ut][ProximityScorer][PhraseFilter]") { + std::vector> positions = {{0}, {}, {2}}; + REQUIRE(sloppy_phrase_match(positions, 100) == false); +} + +TEST_CASE("SloppyPhrase Single Term Passes", "[ut][ProximityScorer][PhraseFilter]") { + std::vector> positions = {{5}}; + REQUIRE(sloppy_phrase_match(positions, 0) == true); +} + +TEST_CASE("SloppyPhrase Multiple Positions Best Window", "[ut][ProximityScorer][PhraseFilter]") { + // A at [0, 10], B at [3, 11]. query offsets [0,1]. + // norms: A → {0, 10}, B → {2, 10}. Window {10,10} → distance=0 → slop=0 passes. + std::vector> positions = {{0, 10}, {3, 11}}; + REQUIRE(sloppy_phrase_match(positions, 0) == true); +} + +// ===== sloppy_phrase_match PosSpan overload (scratch-buffer) tests ===== +// +// Mirrors the vector> cases above but exercises the zero-copy +// PosSpan overload used on the hot path. The two reusable scratch buffers are +// intentionally shared across calls to confirm they are correctly cleared / +// re-sized per invocation. + +TEST_CASE("SloppyPhrase Span Overload Matches Vector Overload", + "[ut][ProximityScorer][PhraseFilter]") { + std::vector norm_buffer; + std::vector count_buffer; + + // Forward distance=2 (same as the vector-overload forward case). + std::vector> forward = {{0}, {2}, {4}}; + auto forward_spans = to_spans(forward); + REQUIRE(sloppy_phrase_match(forward_spans, 2, norm_buffer, count_buffer) == true); + REQUIRE(sloppy_phrase_match(forward_spans, 1, norm_buffer, count_buffer) == false); + + // Reverse costs more slop: distance=6. + std::vector> reverse = {{4}, {2}, {0}}; + auto reverse_spans = to_spans(reverse); + REQUIRE(sloppy_phrase_match(reverse_spans, 6, norm_buffer, count_buffer) == true); + REQUIRE(sloppy_phrase_match(reverse_spans, 5, norm_buffer, count_buffer) == false); + + // Exact adjacent: distance=0. + std::vector> adjacent = {{0}, {1}}; + auto adjacent_spans = to_spans(adjacent); + REQUIRE(sloppy_phrase_match(adjacent_spans, 0, norm_buffer, count_buffer) == true); + + // Multiple positions, best window has distance=0. + std::vector> multi = {{0, 10}, {3, 11}}; + auto multi_spans = to_spans(multi); + REQUIRE(sloppy_phrase_match(multi_spans, 0, norm_buffer, count_buffer) == true); +} + +TEST_CASE("SloppyPhrase Span Overload Missing Term", "[ut][ProximityScorer][PhraseFilter]") { + std::vector norm_buffer; + std::vector count_buffer; + // Middle term has an empty span → no match regardless of slop. + std::vector> positions = {{0}, {}, {2}}; + auto spans = to_spans(positions); + REQUIRE(sloppy_phrase_match(spans, 100, norm_buffer, count_buffer) == false); +} + +TEST_CASE("SloppyPhrase Span Overload Single Term Passes", "[ut][ProximityScorer][PhraseFilter]") { + std::vector norm_buffer; + std::vector count_buffer; + std::vector> positions = {{5}}; + auto spans = to_spans(positions); + REQUIRE(sloppy_phrase_match(spans, 0, norm_buffer, count_buffer) == true); +} + +// ===== extract_positions_from_token_sequence tests ===== + +TEST_CASE("ExtractPositions Basic", "[ut][ProximityScorer]") { + // Document: "苹果(10) 公司(20) 发布(30) 新款(40) 苹果(10) 手机(50) iphone(60)" + uint32_t token_seq[] = {10, 20, 30, 40, 10, 50, 60}; + uint32_t ids[] = {10, 20, 60}; + std::vector> out; + + extract_positions_from_token_sequence(token_seq, 7, ids, 3, 64, out); + + REQUIRE(out.size() == 3); + // term 10 (苹果): positions [0, 4] + REQUIRE(out[0].size() == 2); + REQUIRE(out[0][0] == 0); + REQUIRE(out[0][1] == 4); + // term 20 (公司): positions [1] + REQUIRE(out[1].size() == 1); + REQUIRE(out[1][0] == 1); + // term 60 (iphone): positions [6] + REQUIRE(out[2].size() == 1); + REQUIRE(out[2][0] == 6); +} + +TEST_CASE("ExtractPositions Ignores Non-ID Tokens", "[ut][ProximityScorer]") { + // token_seq has terms 30, 40, 50 that are NOT in ids + uint32_t token_seq[] = {30, 40, 50, 10, 30}; + uint32_t ids[] = {10}; + std::vector> out; + + extract_positions_from_token_sequence(token_seq, 5, ids, 1, 64, out); + + REQUIRE(out.size() == 1); + REQUIRE(out[0].size() == 1); + REQUIRE(out[0][0] == 3); // term 10 at pos 3 +} + +TEST_CASE("ExtractPositions Cap Truncation", "[ut][ProximityScorer]") { + // term 10 appears 10 times, but cap=4 + uint32_t token_seq[] = {10, 10, 10, 10, 10, 10, 10, 10, 10, 10}; + uint32_t ids[] = {10}; + std::vector> out; + + extract_positions_from_token_sequence(token_seq, 10, ids, 1, 4, out); + + REQUIRE(out.size() == 1); + REQUIRE(out[0].size() == 4); // capped at 4 + REQUIRE(out[0][0] == 0); + REQUIRE(out[0][1] == 1); + REQUIRE(out[0][2] == 2); + REQUIRE(out[0][3] == 3); +} + +TEST_CASE("ExtractPositions Null Token Sequence", "[ut][ProximityScorer]") { + uint32_t ids[] = {10, 20}; + std::vector> out; + + extract_positions_from_token_sequence(nullptr, 0, ids, 2, 64, out); + + REQUIRE(out.size() == 2); + REQUIRE(out[0].empty()); + REQUIRE(out[1].empty()); +} + +TEST_CASE("ExtractPositions Empty IDs", "[ut][ProximityScorer]") { + uint32_t token_seq[] = {10, 20, 30}; + std::vector> out; + + extract_positions_from_token_sequence(token_seq, 3, nullptr, 0, 64, out); + + REQUIRE(out.empty()); +} + +TEST_CASE("ExtractPositions Duplicate Term in IDs", "[ut][ProximityScorer]") { + // Same term_id appears twice in ids (unusual but should handle) + uint32_t token_seq[] = {10, 20, 10, 30}; + uint32_t ids[] = {10, 10}; + std::vector> out; + + extract_positions_from_token_sequence(token_seq, 4, ids, 2, 64, out); + + REQUIRE(out.size() == 2); + // Both entries get the same positions + REQUIRE(out[0].size() == 2); + REQUIRE(out[0][0] == 0); + REQUIRE(out[0][1] == 2); + REQUIRE(out[1].size() == 2); + REQUIRE(out[1][0] == 0); + REQUIRE(out[1][1] == 2); +} + +TEST_CASE("ExtractPositions Term Not In Sequence", "[ut][ProximityScorer]") { + // ids contains a term that doesn't appear in token_seq at all + uint32_t token_seq[] = {10, 20, 30}; + uint32_t ids[] = {10, 99}; + std::vector> out; + + extract_positions_from_token_sequence(token_seq, 3, ids, 2, 64, out); + + REQUIRE(out.size() == 2); + REQUIRE(out[0].size() == 1); + REQUIRE(out[0][0] == 0); + REQUIRE(out[1].empty()); // term 99 not found +} + +TEST_CASE("ExtractPositions Caps At Uint16 Range", "[ut][ProximityScorer]") { + std::vector token_seq(65537, 10); + uint32_t ids[] = {10}; + std::vector> out; + + extract_positions_from_token_sequence( + token_seq.data(), static_cast(token_seq.size()), ids, 1, 70000, out); + + REQUIRE(out.size() == 1); + REQUIRE(out[0].size() == 65536); + REQUIRE(out[0].front() == 0); + REQUIRE(out[0].back() == std::numeric_limits::max()); +} diff --git a/src/algorithm/sindi/sindi.cpp b/src/algorithm/sindi/sindi.cpp index 5ccf3b3963..52f23c5f00 100644 --- a/src/algorithm/sindi/sindi.cpp +++ b/src/algorithm/sindi/sindi.cpp @@ -15,6 +15,7 @@ #include "sindi.h" +#include #include #include @@ -200,7 +201,9 @@ SINDI::SINDI(const SINDIParameterPtr& param, const IndexCommonParam& common_para quantization_params_(std::make_shared()), avg_doc_term_length_(param->avg_doc_term_length), remap_term_ids_(param->remap_term_ids), - immutable_enabled_(param->immutable) { + immutable_enabled_(param->immutable), + store_positions_(param->store_positions), + max_positions_per_term_(param->max_positions_per_term) { if (remap_term_ids_) { term_id_mapper_ = std::make_shared(term_id_limit_, common_param.allocator_.get()); @@ -286,7 +289,9 @@ SINDI::Add(const DatasetPtr& base) { term_id_limit_, allocator_, sparse_value_quant_type_, - quantization_params_)); + quantization_params_, + store_positions_, + max_positions_per_term_)); window_changed = true; } @@ -313,6 +318,18 @@ SINDI::Add(const DatasetPtr& base) { try { if (remap_term_ids_) { auto remapped = remap_sparse_vector_for_build(sparse_vector, tmp_ids); + // Remap token_sequence if present: replace original term IDs with compact IDs + Vector remapped_token_seq(allocator_); + if (sparse_vector.token_sequence_ != nullptr && sparse_vector.token_seq_len_ > 0) { + remapped_token_seq.resize(sparse_vector.token_seq_len_); + for (uint32_t ti = 0; ti < sparse_vector.token_seq_len_; ++ti) { + auto mapped = term_id_mapper_->TryMap(sparse_vector.token_sequence_[ti]); + remapped_token_seq[ti] = + mapped.has_value() ? mapped.value() : sparse_vector.token_sequence_[ti]; + } + remapped.token_seq_len_ = sparse_vector.token_seq_len_; + remapped.token_sequence_ = remapped_token_seq.data(); + } window_term_list_[cur_window]->InsertVector(remapped, inner_id); } else { window_term_list_[cur_window]->InsertVector(sparse_vector, inner_id); @@ -414,7 +431,9 @@ SINDI::build_immutable(const DatasetPtr& base) { term_id_limit_, allocator_, sparse_value_quant_type_, - quantization_params_); + quantization_params_, + store_positions_, + max_positions_per_term_); } const auto& sparse_vector = sparse_vectors[i]; @@ -442,6 +461,17 @@ SINDI::build_immutable(const DatasetPtr& base) { try { if (remap_term_ids_) { auto remapped = remap_sparse_vector_for_build(sparse_vector, tmp_ids); + Vector remapped_token_seq(allocator_); + if (sparse_vector.token_sequence_ != nullptr && sparse_vector.token_seq_len_ > 0) { + remapped_token_seq.resize(sparse_vector.token_seq_len_); + for (uint32_t ti = 0; ti < sparse_vector.token_seq_len_; ++ti) { + auto mapped = term_id_mapper_->TryMap(sparse_vector.token_sequence_[ti]); + remapped_token_seq[ti] = + mapped.has_value() ? mapped.value() : sparse_vector.token_sequence_[ti]; + } + remapped.token_seq_len_ = sparse_vector.token_seq_len_; + remapped.token_sequence_ = remapped_token_seq.data(); + } current_window->InsertVector(remapped, inner_id); } else { current_window->InsertVector(sparse_vector, inner_id); @@ -577,12 +607,18 @@ SINDI::KnnSearch(const DatasetPtr& query, auto computer = std::make_shared(effective_query, search_param, allocator_); const SparseVector* rerank_query = (remap_term_ids_ && use_reorder_) ? &sparse_query : nullptr; + ProximityPhraseContext ctx = make_proximity_phrase_context(search_param, effective_query.len_); if (immutable_data_ != nullptr) { - return immutable_search_impl( - computer, inner_param, allocator, UseTermListsHeapInsert(search_param), rerank_query); + return immutable_search_impl(computer, + inner_param, + allocator, + UseTermListsHeapInsert(search_param), + rerank_query, + &ctx); } + return search_impl( - computer, inner_param, allocator, UseTermListsHeapInsert(search_param), rerank_query); + computer, inner_param, allocator, UseTermListsHeapInsert(search_param), rerank_query, &ctx); } std::optional @@ -839,7 +875,8 @@ SINDI::immutable_search_impl(const SparseTermComputerPtr& computer, const InnerSearchParam& inner_param, Allocator* allocator, bool use_term_lists_heap_insert, - const SparseVector* original_query) const { + const SparseVector* original_query, + const ProximityPhraseContext* ctx) const { Allocator* search_allocator = allocator != nullptr ? allocator : allocator_; MaxHeap heap(search_allocator); int64_t k = 0; @@ -851,6 +888,16 @@ SINDI::immutable_search_impl(const SparseTermComputerPtr& computer, ImmutableMappedQueryTerms mapped_terms(search_allocator); auto filter = inner_param.is_inner_id_allowed; const auto [min_window_id, max_window_id] = this->get_min_max_window_id(filter); + + const bool has_phrase = ctx != nullptr && !ctx->phrase_terms.empty(); + const bool has_proximity = ctx != nullptr && ctx->proximity_weight > 0.0f; + // Reusable buffers hoisted out of the window loop, mirroring search_impl. + std::vector> scored_candidates; + std::vector position_lists; + std::vector phrase_spans; + std::vector norm_buffer; + std::vector count_buffer; + for (auto cur = min_window_id; cur <= max_window_id; cur++) { const auto& window = immutable_data_->windows[static_cast(cur)]; const auto window_start_id = static_cast(cur) * window_size_; @@ -858,6 +905,22 @@ SINDI::immutable_search_impl(const SparseTermComputerPtr& computer, std::fill(dists.begin(), dists.end(), 0.0F); scan_immutable_window_by_mapped_terms(dists.data(), window, computer, mapped_terms); + ImmutableWindowPositionBackend pos_backend{this, window, computer->term_retain_ratio_}; + if (store_positions_ && has_phrase) { + phrase_filter(dists.data(), + pos_backend, + ctx->phrase_terms, + ctx->phrase_slop, + phrase_spans, + norm_buffer, + count_buffer); + } + + if (store_positions_ && has_proximity) { + proximity_boost( + dists.data(), computer, pos_backend, scored_candidates, position_lists, *ctx); + } + if (use_term_lists_heap_insert) { if (inner_param.is_inner_id_allowed) { immutable_insert_heap_by_mapped_terms(dists.data(), @@ -945,13 +1008,45 @@ SINDI::immutable_search_impl(const SparseTermComputerPtr& computer, return results; } +ProximityPhraseContext +SINDI::make_proximity_phrase_context(const SINDISearchParameter& search_param, + uint32_t query_term_count) const { + ProximityPhraseContext ctx; + ctx.proximity_weight = search_param.proximity_boost.weight; + ctx.proximity_ordered = search_param.proximity_boost.ordered; + ctx.proximity_candidates = search_param.proximity_boost.candidates; + ctx.proximity_boost_multiplicative = search_param.proximity_boost.boost_multiplicative; + ctx.proximity_all_pairs = search_param.proximity_boost.all_pairs; + ctx.query_term_count = query_term_count; + ctx.phrase_slop = search_param.phrase_slop; + + // Remap phrase terms into the dense internal id space when remapping is + // enabled; unknown terms are dropped. Without remapping the external ids are + // used directly. + if (!search_param.phrase_terms.empty()) { + if (remap_term_ids_ && term_id_mapper_) { + ctx.phrase_terms.reserve(search_param.phrase_terms.size()); + for (auto t : search_param.phrase_terms) { + auto mapped = term_id_mapper_->TryMap(t); + if (mapped.has_value()) { + ctx.phrase_terms.push_back(mapped.value()); + } + } + } else { + ctx.phrase_terms = search_param.phrase_terms; + } + } + return ctx; +} + template DatasetPtr SINDI::search_impl(const SparseTermComputerPtr& computer, const InnerSearchParam& inner_param, Allocator* allocator, bool use_term_lists_heap_insert, - const SparseVector* original_query) const { + const SparseVector* original_query, + const ProximityPhraseContext* ctx) const { // computer and heap MaxHeap heap(allocator); int64_t k = 0; @@ -964,13 +1059,40 @@ SINDI::search_impl(const SparseTermComputerPtr& computer, Vector dists(window_size_, 0.0, allocator); auto filter = inner_param.is_inner_id_allowed; const auto [min_window_id, max_window_id] = this->get_min_max_window_id(filter); + + const bool has_phrase = ctx != nullptr && !ctx->phrase_terms.empty(); + const bool has_proximity = ctx != nullptr && ctx->proximity_weight > 0.0f; + + // Reusable buffers for proximity/phrase position lookup. + std::vector> scored_candidates; + std::vector position_lists; + std::vector phrase_spans; + std::vector norm_buffer; + std::vector count_buffer; + for (auto cur = min_window_id; cur <= max_window_id; cur++) { auto window_start_id = cur * window_size_; auto term_list = this->window_term_list_[cur]; - // compute + // compute IP scores term_list->Query(dists.data(), computer); + DatacellPositionBackend pos_backend{term_list, computer->term_retain_ratio_}; + if (store_positions_ && has_phrase) { + phrase_filter(dists.data(), + pos_backend, + ctx->phrase_terms, + ctx->phrase_slop, + phrase_spans, + norm_buffer, + count_buffer); + } + + if (store_positions_ && has_proximity) { + proximity_boost( + dists.data(), computer, pos_backend, scored_candidates, position_lists, *ctx); + } + // insert heap if (use_term_lists_heap_insert) { if (inner_param.is_inner_id_allowed) { @@ -991,7 +1113,6 @@ SINDI::search_impl(const SparseTermComputerPtr& computer, } } - // rerank if (use_reorder_) { // high precision float cur_heap_top = std::numeric_limits::max(); @@ -1056,6 +1177,120 @@ SINDI::search_impl(const SparseTermComputerPtr& computer, return results; } +template +void +SINDI::phrase_filter(float* dists, + const PosBackend& pos, + const std::vector& phrase_terms, + uint32_t phrase_slop, + std::vector& phrase_spans, + std::vector& norm_buffer, + std::vector& count_buffer) const { + // Resolve each phrase term once; the global->local translation is invariant + // across the window, so it runs phrase_terms.size() times instead of + // window_size_ times. + std::vector phrase_views(phrase_terms.size()); + for (size_t i = 0; i < phrase_terms.size(); ++i) { + phrase_views[i] = pos.ResolveTerm(phrase_terms[i]); + } + for (uint32_t doc_idx = 0; doc_idx < window_size_; ++doc_idx) { + if (dists[doc_idx] >= 0.0f) { + continue; + } + // Collect zero-copy position views for phrase terms in this doc. Each + // view is located by binary search over the term's retained posting + // prefix, so a doc missing any phrase term is discarded. + phrase_spans.clear(); + bool all_present = true; + for (const auto& view : phrase_views) { + PosSpan span = pos.LookupPositions(view, static_cast(doc_idx)); + if (span.empty()) { + all_present = false; + break; + } + phrase_spans.push_back(span); + } + bool phrase_ok = all_present && + sloppy_phrase_match(phrase_spans, phrase_slop, norm_buffer, count_buffer); + if (!phrase_ok) { + dists[doc_idx] = 0.0f; // discard + } + } +} + +template +void +SINDI::proximity_boost(float* dists, + const SparseTermComputerPtr& computer, + const PosBackend& pos, + std::vector>& scored_candidates, + std::vector& position_lists, + const ProximityPhraseContext& ctx) const { + const auto& raw_query = computer->raw_query_; + const uint32_t proximity_candidates = ctx.proximity_candidates; + const bool proximity_ordered = ctx.proximity_ordered; + const bool proximity_all_pairs = ctx.proximity_all_pairs; + const bool proximity_boost_multiplicative = ctx.proximity_boost_multiplicative; + const float proximity_weight = ctx.proximity_weight; + const uint32_t query_term_count = ctx.query_term_count; + + // Collect candidate doc_ids with non-zero scores, then keep top-N by IP score + // Keep top-proximity_candidates by smallest dist (most negative = highest IP) + scored_candidates.clear(); + for (uint32_t doc_idx = 0; doc_idx < window_size_; ++doc_idx) { + if (dists[doc_idx] < 0.0f) { + scored_candidates.emplace_back(dists[doc_idx], doc_idx); + } + } + if (scored_candidates.size() > proximity_candidates) { + std::nth_element(scored_candidates.begin(), + scored_candidates.begin() + proximity_candidates, + scored_candidates.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + scored_candidates.resize(proximity_candidates); + } + + // Pre-resolve each query term to a reusable posting-list view once, outside + // the candidate loop, so the global->local translation is paid query_len + // times instead of candidates*query_len times. + std::vector views(raw_query.len_); + for (uint32_t qi = 0; qi < raw_query.len_; ++qi) { + views[qi] = pos.ResolveTerm(raw_query.ids_[qi]); + } + + // For each candidate, fetch positions by binary search over each query + // term's retained posting prefix. PosSpan elements point into the backend's + // position pool, which is read-only for the duration of this query, so the + // zero-copy views stay valid until consumed. + for (const auto& cand : scored_candidates) { + uint32_t doc_idx = cand.second; + position_lists.clear(); + + for (uint32_t qi = 0; qi < raw_query.len_; ++qi) { + position_lists.push_back( + pos.LookupPositions(views[qi], static_cast(doc_idx))); + } + + float raw_boost = proximity_all_pairs + ? all_pairwise_proximity(position_lists, proximity_ordered) + : adjacent_pairwise_proximity(position_lists, proximity_ordered); + if (raw_boost > 0.0f) { + // Normalize by the number of scored pairs: C(query_term_count, 2) for + // all-pairs, n-1 for adjacent-only. + float pair_count = proximity_all_pairs + ? static_cast(query_term_count) * + static_cast(query_term_count - 1) / 2.0f + : static_cast(query_term_count - 1); + float normalized_boost = (pair_count > 0.0f) ? raw_boost / pair_count : 0.0f; + if (proximity_boost_multiplicative) { + dists[doc_idx] *= (1.0f + proximity_weight * normalized_boost); + } else { + dists[doc_idx] -= proximity_weight * normalized_boost; + } + } + } +} + DatasetPtr SINDI::RangeSearch(const DatasetPtr& query, float radius, @@ -1095,12 +1330,22 @@ SINDI::RangeSearch(const DatasetPtr& query, auto computer = std::make_shared(effective_query, search_param, allocator_); const SparseVector* rerank_query = (remap_term_ids_ && use_reorder_) ? &sparse_query : nullptr; + ProximityPhraseContext ctx = make_proximity_phrase_context(search_param, effective_query.len_); if (immutable_data_ != nullptr) { - return immutable_search_impl( - computer, inner_param, allocator_, UseTermListsHeapInsert(search_param), rerank_query); - } - return search_impl( - computer, inner_param, allocator_, UseTermListsHeapInsert(search_param), rerank_query); + return immutable_search_impl(computer, + inner_param, + allocator_, + UseTermListsHeapInsert(search_param), + rerank_query, + &ctx); + } + + return search_impl(computer, + inner_param, + allocator_, + UseTermListsHeapInsert(search_param), + rerank_query, + &ctx); } bool @@ -1123,6 +1368,8 @@ SINDI::cal_memory_usage() { memory += window.offsets.size() * sizeof(uint32_t); memory += window.id_payloads.size() * sizeof(uint16_t); memory += window.value_payloads.size() * sizeof(uint8_t); + memory += window.pos_offsets.size() * sizeof(uint32_t); + memory += window.pos_pool.size() * sizeof(uint16_t); } } else { memory += window_term_list_.size() * sizeof(SparseTermDataCellPtr); @@ -1341,7 +1588,9 @@ SINDI::deserialize_windows(StreamReader& reader_ref) { term_id_limit_, allocator_, sparse_value_quant_type_, - quantization_params_); + quantization_params_, + store_positions_, + max_positions_per_term_); window->Deserialize(reader_ref); } } @@ -1563,7 +1812,9 @@ SINDI::Deserialize(StreamReader& reader) { term_id_limit_, allocator_, sparse_value_quant_type_, - quantization_params_); + quantization_params_, + store_positions_, + max_positions_per_term_); window->Deserialize(reader_ref); } } @@ -1599,6 +1850,8 @@ SINDI::compact_window_to_immutable(const SparseTermDataCell& term_list, window.id_payloads.clear(); window.value_payloads.clear(); window.sorted_global_terms.clear(); + window.pos_offsets.clear(); + window.pos_pool.clear(); if (not remap_term_ids_) { window.offsets.reserve(static_cast(term_list.term_capacity_) + 1); @@ -1638,9 +1891,136 @@ SINDI::compact_window_to_immutable(const SparseTermDataCell& term_list, term_list.term_datas_[term]->data(), payload_bytes); + if (store_positions_) { + for (uint32_t within = 0; within < posting_count; ++within) { + auto [positions, positions_size] = term_list.GetPositionsView(term, within); + if (window.pos_pool.size() > std::numeric_limits::max()) { + throw VsagException(ErrorType::INVALID_ARGUMENT, + "immutable SINDI position pool overflows uint32_t"); + } + window.pos_offsets.push_back(static_cast(window.pos_pool.size())); + if (positions_size == 0) { + continue; + } + if (positions == nullptr) { + throw VsagException(ErrorType::INVALID_ARGUMENT, + "immutable SINDI position view is null"); + } + const uint64_t pool_size = window.pos_pool.size(); + if (positions_size > std::numeric_limits::max() - pool_size) { + throw VsagException(ErrorType::INVALID_ARGUMENT, + "immutable SINDI position pool overflows uint32_t"); + } + const auto old_pos_size = window.pos_pool.size(); + window.pos_pool.resize(old_pos_size + positions_size); + std::memcpy(window.pos_pool.data() + old_pos_size, + positions, + positions_size * sizeof(uint16_t)); + } + } + total_posting_count += posting_count; window.offsets.push_back(static_cast(total_posting_count)); } + if (store_positions_) { + CHECK_ARGUMENT(window.pos_pool.size() <= std::numeric_limits::max(), + "immutable SINDI position pool overflows uint32_t"); + window.pos_offsets.push_back(static_cast(window.pos_pool.size())); + CHECK_ARGUMENT(window.pos_offsets.size() == window.id_payloads.size() + 1, + "immutable SINDI position offsets size does not match postings"); + } +} + +void +SINDI::serialize_immutable_window_positions(StreamWriter& writer, + const ImmutableSINDIWindow& window) const { + uint32_t term_capacity = 0; + if (remap_term_ids_) { + if (not window.sorted_global_terms.empty()) { + CHECK_ARGUMENT(window.sorted_global_terms.back() < term_id_limit_, + "immutable SINDI remapped term exceeds term_id_limit"); + term_capacity = window.sorted_global_terms.back() + 1; + } + } else if (not window.offsets.empty()) { + term_capacity = static_cast(window.offsets.size() - 1); + } + + Vector term_sizes(term_capacity, 0, allocator_); + if (remap_term_ids_) { + for (uint32_t local_term = 0; local_term < window.sorted_global_terms.size(); + ++local_term) { + const auto term = window.sorted_global_terms[local_term]; + CHECK_ARGUMENT(local_term + 1 < window.offsets.size(), + "immutable SINDI term and offset counts do not match"); + term_sizes[term] = window.offsets[local_term + 1] - window.offsets[local_term]; + } + } else { + for (uint32_t term = 0; term < term_capacity; ++term) { + term_sizes[term] = window.offsets[term + 1] - window.offsets[term]; + } + } + + StreamWriter::WriteObj(writer, static_cast(1)); + + Vector empty_offsets(allocator_); + Vector empty_pool(allocator_); + Vector term_offsets(allocator_); + Vector term_pool(allocator_); + + uint32_t next_remap_pos = 0; + for (uint32_t term = 0; term < term_capacity; ++term) { + std::optional local_term; + if (remap_term_ids_) { + if (next_remap_pos < window.sorted_global_terms.size() && + window.sorted_global_terms[next_remap_pos] == term) { + local_term = next_remap_pos; + ++next_remap_pos; + } + } else { + local_term = term; + } + + if (not local_term.has_value() || term_sizes[term] == 0 || window.pos_offsets.empty()) { + StreamWriter::WriteVector(writer, empty_offsets); + StreamWriter::WriteVector(writer, empty_pool); + continue; + } + + const auto begin = window.offsets[local_term.value()]; + const auto end = window.offsets[local_term.value() + 1]; + CHECK_ARGUMENT(term_sizes[term] == end - begin, + "immutable SINDI term_sizes are inconsistent"); + + term_offsets.clear(); + term_pool.clear(); + term_offsets.reserve(term_sizes[term]); + uint32_t pool_cursor = 0; + for (uint32_t p = begin; p < end; ++p) { + CHECK_ARGUMENT(p + 1 < window.pos_offsets.size(), + "immutable SINDI position offsets length mismatch"); + const uint32_t start = window.pos_offsets[p]; + const uint32_t pos_end = window.pos_offsets[p + 1]; + CHECK_ARGUMENT(start <= pos_end, "immutable SINDI position offsets are unordered"); + CHECK_ARGUMENT(pos_end <= window.pos_pool.size(), + "immutable SINDI position offsets exceed pool size"); + + term_offsets.push_back(pool_cursor); + const uint32_t span_size = pos_end - start; + CHECK_ARGUMENT(span_size <= std::numeric_limits::max() - pool_cursor, + "immutable SINDI term position pool overflows uint32_t"); + const auto old_pool_size = term_pool.size(); + term_pool.resize(old_pool_size + span_size); + if (span_size > 0) { + std::memcpy(term_pool.data() + old_pool_size, + window.pos_pool.data() + start, + span_size * sizeof(uint16_t)); + } + pool_cursor += span_size; + } + + StreamWriter::WriteVector(writer, term_offsets); + StreamWriter::WriteVector(writer, term_pool); + } } void @@ -1649,6 +2029,92 @@ SINDI::serialize_immutable_window(StreamWriter& writer, const ImmutableSINDIWind StreamWriter::WriteVector(writer, window.offsets); StreamWriter::WriteVector(writer, window.id_payloads); StreamWriter::WriteVector(writer, window.value_payloads); + + if (store_positions_) { + serialize_immutable_window_positions(writer, window); + } +} + +void +SINDI::deserialize_immutable_window_positions(StreamReader& reader_ref, + ImmutableSINDIWindow& window) const { + const auto posting_count = static_cast(window.offsets.back()); + uint32_t term_capacity = 0; + Vector term_sizes(allocator_); + if (remap_term_ids_) { + if (not window.sorted_global_terms.empty()) { + CHECK_ARGUMENT(window.sorted_global_terms.back() < term_id_limit_, + "immutable SINDI remapped term exceeds term_id_limit"); + term_capacity = window.sorted_global_terms.back() + 1; + } + term_sizes.resize(term_capacity); + for (uint32_t local_term = 0; local_term < window.sorted_global_terms.size(); + ++local_term) { + const auto term = window.sorted_global_terms[local_term]; + term_sizes[term] = window.offsets[local_term + 1] - window.offsets[local_term]; + } + } else { + term_capacity = static_cast(window.offsets.size() - 1); + term_sizes.resize(term_capacity); + for (uint32_t term = 0; term < term_capacity; ++term) { + term_sizes[term] = window.offsets[term + 1] - window.offsets[term]; + } + } + + uint8_t marker = 0; + StreamReader::ReadObj(reader_ref, marker); + CHECK_ARGUMENT(marker == 1, "immutable SINDI position block marker mismatch"); + window.pos_offsets.reserve(posting_count + 1); + Vector offsets_buf(allocator_); + Vector pool_buf(allocator_); + uint64_t pool_cursor = 0; + for (uint32_t term = 0; term < term_capacity; ++term) { + StreamReader::ReadVector(reader_ref, offsets_buf); + StreamReader::ReadVector(reader_ref, pool_buf); + const auto term_posting_count = term_sizes[term]; + if (term_posting_count == 0) { + CHECK_ARGUMENT(offsets_buf.empty() && pool_buf.empty(), + "immutable SINDI empty term has position payload"); + continue; + } + if (offsets_buf.empty()) { + CHECK_ARGUMENT(pool_buf.empty(), + "immutable SINDI position pool exists without offsets"); + for (uint32_t within = 0; within < term_posting_count; ++within) { + window.pos_offsets.push_back(static_cast(pool_cursor)); + } + continue; + } + CHECK_ARGUMENT(offsets_buf.size() == term_posting_count, + "immutable SINDI position offsets length mismatch"); + CHECK_ARGUMENT(offsets_buf.front() == 0, + "immutable SINDI position offsets must start at zero"); + for (uint32_t within = 0; within < term_posting_count; ++within) { + CHECK_ARGUMENT(offsets_buf[within] <= pool_buf.size(), + "immutable SINDI position offset exceeds pool size"); + if (within > 0) { + CHECK_ARGUMENT(offsets_buf[within - 1] <= offsets_buf[within], + "immutable SINDI position offsets are unordered"); + } + CHECK_ARGUMENT(offsets_buf[within] <= + std::numeric_limits::max() - pool_cursor, + "immutable SINDI position offset overflows uint32_t"); + window.pos_offsets.push_back(static_cast(pool_cursor + offsets_buf[within])); + } + CHECK_ARGUMENT(pool_buf.size() <= std::numeric_limits::max() - pool_cursor, + "immutable SINDI position pool overflows uint32_t"); + const auto old_pool_size = window.pos_pool.size(); + window.pos_pool.resize(old_pool_size + pool_buf.size()); + std::memcpy(window.pos_pool.data() + old_pool_size, + pool_buf.data(), + pool_buf.size() * sizeof(uint16_t)); + pool_cursor += pool_buf.size(); + } + window.pos_offsets.push_back(static_cast(pool_cursor)); + CHECK_ARGUMENT(window.pos_offsets.size() == window.id_payloads.size() + 1, + "immutable SINDI position offsets size does not match postings"); + CHECK_ARGUMENT(window.pos_pool.size() == pool_cursor, + "immutable SINDI position pool size mismatch"); } void @@ -1693,6 +2159,8 @@ SINDI::deserialize_immutable_window(StreamReader& reader_ref, ImmutableSINDIWind window.sorted_global_terms.end(), std::greater_equal()) == window.sorted_global_terms.end(), "immutable SINDI remapped terms must be strictly increasing"); + CHECK_ARGUMENT(window.offsets.size() == window.sorted_global_terms.size() + 1, + "immutable SINDI term and offset counts do not match"); } else { CHECK_ARGUMENT(window.sorted_global_terms.empty(), "immutable SINDI dense window must not contain remapped terms"); @@ -1721,6 +2189,10 @@ SINDI::deserialize_immutable_window(StreamReader& reader_ref, ImmutableSINDIWind CHECK_ARGUMENT(id < window_size_, "immutable SINDI window-local doc id exceeds window size"); } + + if (store_positions_) { + deserialize_immutable_window_positions(reader_ref, window); + } } std::pair diff --git a/src/algorithm/sindi/sindi.h b/src/algorithm/sindi/sindi.h index 315e63f716..592bfe5509 100644 --- a/src/algorithm/sindi/sindi.h +++ b/src/algorithm/sindi/sindi.h @@ -15,9 +15,14 @@ #pragma once +#include #include +#include +#include +#include #include "algorithm/inner_index_interface.h" +#include "algorithm/sindi/proximity_scorer.h" #include "algorithm/sindi/term_id_mapper.h" #include "datacell/flatten_interface.h" #include "datacell/sparse_term_datacell.h" @@ -30,13 +35,22 @@ struct ImmutableSINDIWindow { : sorted_global_terms(allocator), offsets(allocator), id_payloads(allocator), - value_payloads(allocator) { + value_payloads(allocator), + pos_offsets(allocator), + pos_pool(allocator) { } Vector sorted_global_terms; Vector offsets; Vector id_payloads; Vector value_payloads; + // Position storage (only populated when store_positions_ is enabled). Flat + // per-window layout aligned to `offsets`: for a flat posting index + // p = offsets[local_term] + within, its positions are + // pos_pool[pos_offsets[p] .. pos_offsets[p+1]). pos_offsets has a trailing + // sentinel so pos_offsets[p+1] is always valid; size == offsets.back()+1. + Vector pos_offsets; + Vector pos_pool; }; struct ImmutableSINDIData { @@ -48,6 +62,25 @@ struct ImmutableSINDIData { Vector windows; }; +/** + * @brief Aggregated proximity/phrase parameters passed through the search path. + * + * Bundles the proximity/phrase scalars plus two derived values that are NOT raw + * SINDISearchParameter fields: `query_term_count` (the remapped query length) + * and `phrase_terms` (the remapped phrase terms). Built once at the KNN/RANGE + * dispatch site and forwarded by const-ref to search_impl / immutable_search_impl. + */ +struct ProximityPhraseContext { + float proximity_weight{0.0f}; + bool proximity_ordered{false}; + uint32_t proximity_candidates{10000}; + bool proximity_boost_multiplicative{true}; + bool proximity_all_pairs{false}; + uint32_t query_term_count{0}; + std::vector phrase_terms; + uint32_t phrase_slop{0}; +}; + /** * @brief SINDI: Sparse INverted Index with windowed term lists. * @@ -208,7 +241,220 @@ class SINDI : public InnerIndexInterface { const InnerSearchParam& inner_param, Allocator* allocator, bool use_term_lists_heap_insert, - const SparseVector* original_query = nullptr) const; + const SparseVector* original_query = nullptr, + const ProximityPhraseContext* ctx = nullptr) const; + + /** + * @brief Position accessor over the mutable per-term datacell layout. + * + * Nested so it can reach SparseTermDataCell's public accessors while keeping + * the type private to SINDI. + */ + struct DatacellPositionBackend { + const SparseTermDataCellPtr& term_list; + float term_retain_ratio{1.0f}; + + // Read-only view of one query term's posting list, produced by + // ResolveTerm and reused across candidate docs by LookupPositions. + // valid : false when the term is absent (or stores no positions); + // such a view yields empty spans for any doc. + // id_begin : (*term_ids_[global]).data(), start of the doc-id list. + // retained : term_sizes_[global] * term_retain_ratio, the prefix to + // binary-search. + // global_term : needed by GetPositionsView on the hit path. + struct TermPostingListView { + bool valid{false}; + const uint16_t* id_begin{nullptr}; + uint32_t retained{0}; + uint32_t global_term{0}; + }; + + // Resolve one query term to a reusable posting-list view: guard checks + + // capture the doc-id list pointer and retained prefix length. + TermPostingListView + ResolveTerm(uint32_t global_term) const { + TermPostingListView view; + if (global_term >= term_list->term_capacity_ || + term_list->term_sizes_[global_term] == 0 || !term_list->term_ids_[global_term] || + !term_list->term_pos_offsets_[global_term]) { + return view; // valid stays false + } + const auto& ids = *term_list->term_ids_[global_term]; + view.id_begin = ids.data(); + view.retained = static_cast( + static_cast(term_list->term_sizes_[global_term]) * term_retain_ratio); + view.global_term = global_term; + view.valid = true; + return view; + } + + // Look up positions for a doc using a pre-resolved posting-list view. + // Binary-searches doc_id in the view's retained posting prefix; the + // retained prefix mirrors the scan's term_retain_ratio pruning, so a doc + // present only in the pruned tail reads as absent. Returns {nullptr, 0} + // when the term is absent from the doc (or has no stored positions). + PosSpan + LookupPositions(const TermPostingListView& view, uint16_t doc_id) const { + if (!view.valid) { + return PosSpan{nullptr, 0}; + } + const auto* begin = view.id_begin; + const auto* end = begin + view.retained; + const auto* found = std::lower_bound(begin, end, doc_id); + if (found == end || *found != doc_id) { + return PosSpan{nullptr, 0}; + } + return GetPositionsView(view.global_term, static_cast(found - begin)); + } + + private: + // Return the position list stored at a known slot in a term's posting. + // global_term : the term whose posting we index into. + // within : the slot index within that posting (0-based rank). + PosSpan + GetPositionsView(uint32_t global_term, uint32_t within) const { + auto [ptr, size] = term_list->GetPositionsView(global_term, within); + return PosSpan{ptr, size}; + } + }; + + /** + * @brief Position accessor over the immutable flat per-window layout. + * + * Nested so it can call SINDI's private get_immutable_local_term (global -> + * local term translation, honoring remap). + */ + struct ImmutableWindowPositionBackend { + const SINDI* self; + const ImmutableSINDIWindow& window; + float term_retain_ratio{1.0f}; + + // Read-only view of one query term's inverted list in this window, + // produced by ResolveTerm and reused across candidate docs by + // LookupPositions. Resolving once outside the candidate loop pays the + // global->local translation query_len times instead of + // candidates*query_len times. + // valid : false when the term is absent from this window (or the + // window stores no positions); such a view yields empty spans. + // id_begin : id_payloads.data() + begin_doc, start of the doc-id list. + // retained : length of the retained posting prefix to binary-search. + // flat_base: offsets[local], so a doc at prefix rank r maps to flat + // posting index flat_base + r. + struct TermPostingListView { + bool valid{false}; + const uint16_t* id_begin{nullptr}; + uint32_t retained{0}; + uint32_t flat_base{0}; + }; + + // Resolve one query term to a reusable posting-list view (one + // global->local translation + posting-range capture). See + // TermPostingListView. + TermPostingListView + ResolveTerm(uint32_t global_term) const { + TermPostingListView view; + if (window.pos_offsets.empty()) { + return view; // valid stays false + } + auto local = self->get_immutable_local_term(window, global_term); + if (!local.has_value()) { + return view; // term absent from this window + } + const uint32_t begin_doc = window.offsets[local.value()]; + const uint32_t end_doc = window.offsets[local.value() + 1]; + view.retained = + static_cast(static_cast(end_doc - begin_doc) * term_retain_ratio); + view.id_begin = window.id_payloads.data() + begin_doc; + view.flat_base = begin_doc; + view.valid = true; + return view; + } + + // Look up positions for a doc using a pre-resolved posting-list view. + // Only the doc binary search + position read remain in the candidate + // loop; the global->local translation was already paid by ResolveTerm. + // Binary-searches doc_id in the view's retained posting prefix; the + // retained prefix mirrors the scan's term_retain_ratio pruning, so a doc + // present only in the pruned tail reads as absent. Returns {nullptr, 0} + // when the term is absent from the doc (or has no stored positions). + PosSpan + LookupPositions(const TermPostingListView& view, uint16_t doc_id) const { + if (!view.valid) { + return PosSpan{nullptr, 0}; + } + const auto* begin = view.id_begin; + const auto* end = begin + view.retained; + const auto* found = std::lower_bound(begin, end, doc_id); + if (found == end || *found != doc_id) { + return PosSpan{nullptr, 0}; + } + return positions_at_flat_index(view.flat_base + static_cast(found - begin)); + } + + private: + // Return the position list stored at a flat posting index (no + // global->local translation). + PosSpan + positions_at_flat_index(uint32_t p) const { + const uint32_t start = window.pos_offsets[p]; + const uint32_t end = window.pos_offsets[p + 1]; + return PosSpan{window.pos_pool.data() + start, end - start}; + } + }; + + /** + * @brief Phrase filter: for candidates in this window with dists < 0, apply + * the sloppy phrase constraint; those failing are set to dists = 0. + * + * Phrase terms are resolved once via pos.ResolveTerm before the doc loop; + * per-doc positions are then fetched via pos.LookupPositions (binary search + * over the retained posting prefix), so no reverse offset map is needed. + * phrase_spans / norm_buffer / count_buffer are reused buffers hoisted out + * of the window loop by the caller. + * + * @tparam PosBackend supplies ResolveTerm(global_term) -> TermPostingListView + * and LookupPositions(view, doc_id) -> PosSpan; a + * {nullptr, 0} span means the term has no positions in + * that doc. Backends wrap either the mutable datacell or + * an immutable window. + */ + template + void + phrase_filter(float* dists, + const PosBackend& pos, + const std::vector& phrase_terms, + uint32_t phrase_slop, + std::vector& phrase_spans, + std::vector& norm_buffer, + std::vector& count_buffer) const; + + /** + * @brief Proximity boost: adjust dists for the top-N candidates in this + * window by their term proximity. scored_candidates / position_lists + * are reused buffers hoisted out of the window loop by the caller. + * Query terms are resolved once via pos.ResolveTerm, then positions + * are fetched per candidate via pos.LookupPositions. + * + * @tparam PosBackend see phrase_filter. + */ + template + void + proximity_boost(float* dists, + const SparseTermComputerPtr& computer, + const PosBackend& pos, + std::vector>& scored_candidates, + std::vector& position_lists, + const ProximityPhraseContext& ctx) const; + + /** + * @brief Build the proximity/phrase context for a search, remapping phrase + * terms and capturing the (post-remap) query length. Shared by the + * KNN and RANGE dispatch sites so the mutable and immutable paths get + * identical derived values. + */ + ProximityPhraseContext + make_proximity_phrase_context(const SINDISearchParameter& search_param, + uint32_t query_term_count) const; template DatasetPtr @@ -216,7 +462,8 @@ class SINDI : public InnerIndexInterface { const InnerSearchParam& inner_param, Allocator* allocator, bool use_term_lists_heap_insert, - const SparseVector* original_query = nullptr) const; + const SparseVector* original_query = nullptr, + const ProximityPhraseContext* ctx = nullptr) const; bool UseTermListsHeapInsert(const SINDISearchParameter& search_param) const; @@ -261,6 +508,14 @@ class SINDI : public InnerIndexInterface { void serialize_immutable_window(StreamWriter& writer, const ImmutableSINDIWindow& window) const; + void + serialize_immutable_window_positions(StreamWriter& writer, + const ImmutableSINDIWindow& window) const; + + void + deserialize_immutable_window_positions(StreamReader& reader_ref, + ImmutableSINDIWindow& window) const; + void compact_window_to_immutable(const SparseTermDataCell& term_list, ImmutableSINDIWindow& window) const; @@ -374,6 +629,9 @@ class SINDI : public InnerIndexInterface { bool immutable_enabled_{false}; std::unique_ptr immutable_data_{nullptr}; + + bool store_positions_{false}; + uint32_t max_positions_per_term_{64}; }; } // namespace vsag diff --git a/src/algorithm/sindi/sindi_parameter.cpp b/src/algorithm/sindi/sindi_parameter.cpp index f44f78a575..8f56df6a43 100644 --- a/src/algorithm/sindi/sindi_parameter.cpp +++ b/src/algorithm/sindi/sindi_parameter.cpp @@ -130,6 +130,17 @@ SINDIParameter::FromJson(const JsonType& json) { if (json.Contains(SPARSE_IMMUTABLE)) { immutable = json[SPARSE_IMMUTABLE].GetBool(); } + + if (json.Contains(SPARSE_STORE_POSITIONS)) { + store_positions = json[SPARSE_STORE_POSITIONS].GetBool(); + } + + if (json.Contains(SPARSE_MAX_POSITIONS_PER_TERM)) { + max_positions_per_term = json[SPARSE_MAX_POSITIONS_PER_TERM].GetInt(); + CHECK_ARGUMENT((0 < max_positions_per_term and max_positions_per_term <= 256), + fmt::format("max_positions_per_term must in (0, 256], but now is {}", + max_positions_per_term)); + } } JsonType @@ -149,6 +160,8 @@ SINDIParameter::ToJson() const { if (immutable) { json[SPARSE_IMMUTABLE].SetBool(true); } + json[SPARSE_STORE_POSITIONS].SetBool(store_positions); + json[SPARSE_MAX_POSITIONS_PER_TERM].SetInt(max_positions_per_term); return json; } @@ -163,6 +176,8 @@ SINDIParameter::CheckCompatibility(const vsag::ParamPtr& other) const { CHECK_FIELD_EQ(*this, *p, avg_doc_term_length); CHECK_FIELD_EQ(*this, *p, remap_term_ids); CHECK_FIELD_EQ(*this, *p, immutable); + CHECK_FIELD_EQ(*this, *p, store_positions); + CHECK_FIELD_EQ(*this, *p, max_positions_per_term); return true; } @@ -198,6 +213,52 @@ SINDISearchParameter::FromJson(const JsonType& json) { "Remove this key; heap insertion is derived from doc_prune_ratio " "and query_prune_ratio with the current SINDI prune-ratio threshold"); } + + const auto& sindi_json = json[INDEX_SINDI]; + if (sindi_json.Contains(SPARSE_PROXIMITY_BOOST)) { + const auto& proximity_json = sindi_json[SPARSE_PROXIMITY_BOOST]; + CHECK_ARGUMENT(proximity_json.IsObject(), + fmt::format("{} must be an object", SPARSE_PROXIMITY_BOOST)); + + if (proximity_json.Contains(SPARSE_PROXIMITY_BOOST_CANDIDATES)) { + auto candidates = proximity_json[SPARSE_PROXIMITY_BOOST_CANDIDATES].GetInt(); + CHECK_ARGUMENT(candidates > 0, + fmt::format("proximity_boost.candidates must be > 0, got {}", candidates)); + proximity_boost.candidates = static_cast(candidates); + } + + if (proximity_json.Contains(SPARSE_PROXIMITY_BOOST_WEIGHT)) { + proximity_boost.weight = proximity_json[SPARSE_PROXIMITY_BOOST_WEIGHT].GetFloat(); + } + + if (proximity_json.Contains(SPARSE_PROXIMITY_BOOST_ORDERED)) { + proximity_boost.ordered = proximity_json[SPARSE_PROXIMITY_BOOST_ORDERED].GetBool(); + } + + if (proximity_json.Contains(SPARSE_PROXIMITY_BOOST_MULTIPLICATIVE)) { + proximity_boost.boost_multiplicative = + proximity_json[SPARSE_PROXIMITY_BOOST_MULTIPLICATIVE].GetBool(); + } + + if (proximity_json.Contains(SPARSE_PROXIMITY_BOOST_ALL_PAIRS)) { + proximity_boost.all_pairs = + proximity_json[SPARSE_PROXIMITY_BOOST_ALL_PAIRS].GetBool(); + } + } + + if (json[INDEX_SINDI].Contains(SPARSE_PHRASE_TERMS) && + json[INDEX_SINDI][SPARSE_PHRASE_TERMS].IsArray()) { + auto terms_i32 = json[INDEX_SINDI][SPARSE_PHRASE_TERMS].GetVector(); + phrase_terms.clear(); + phrase_terms.reserve(terms_i32.size()); + for (auto t : terms_i32) { + phrase_terms.push_back(static_cast(t)); + } + } + + if (json[INDEX_SINDI].Contains(SPARSE_PHRASE_SLOP)) { + phrase_slop = json[INDEX_SINDI][SPARSE_PHRASE_SLOP].GetInt(); + } } JsonType SINDISearchParameter::ToJson() const { @@ -206,6 +267,18 @@ SINDISearchParameter::ToJson() const { json[INDEX_SINDI][SPARSE_QUERY_PRUNE_RATIO].SetFloat(query_prune_ratio); json[INDEX_SINDI][SPARSE_N_CANDIDATE].SetInt(n_candidate); json[INDEX_SINDI][SPARSE_TERM_PRUNE_RATIO].SetFloat(term_prune_ratio); + json[INDEX_SINDI][SPARSE_PROXIMITY_BOOST].SetJson(JsonType()); + auto&& proximity_json = json[INDEX_SINDI][SPARSE_PROXIMITY_BOOST]; + proximity_json[SPARSE_PROXIMITY_BOOST_CANDIDATES].SetInt(proximity_boost.candidates); + proximity_json[SPARSE_PROXIMITY_BOOST_WEIGHT].SetFloat(proximity_boost.weight); + proximity_json[SPARSE_PROXIMITY_BOOST_ORDERED].SetBool(proximity_boost.ordered); + proximity_json[SPARSE_PROXIMITY_BOOST_MULTIPLICATIVE].SetBool( + proximity_boost.boost_multiplicative); + proximity_json[SPARSE_PROXIMITY_BOOST_ALL_PAIRS].SetBool(proximity_boost.all_pairs); + if (not phrase_terms.empty()) { + json[INDEX_SINDI][SPARSE_PHRASE_TERMS].SetVector(phrase_terms); + } + json[INDEX_SINDI][SPARSE_PHRASE_SLOP].SetInt(phrase_slop); return json; } diff --git a/src/algorithm/sindi/sindi_parameter.h b/src/algorithm/sindi/sindi_parameter.h index e7e053a362..39aa84a60d 100644 --- a/src/algorithm/sindi/sindi_parameter.h +++ b/src/algorithm/sindi/sindi_parameter.h @@ -63,6 +63,10 @@ class SINDIParameter : public InnerIndexParameter { bool immutable{false}; + bool store_positions{false}; + + uint32_t max_positions_per_term{64}; + // temporal parameter bool deserialize_without_footer{false}; bool deserialize_without_buffer{false}; @@ -80,12 +84,36 @@ class SINDISearchParameter : public Parameter { SINDISearchParameter() = default; public: + struct ProximityBoostParameter { + // Weight of the proximity boost. Zero disables proximity scoring. + float weight{0.0f}; + + // Top N candidates participating in proximity scoring within a single window + uint32_t candidates{10000}; + + // true → all pairs C(n,2), false → adjacent-only pairs (n-1) + bool all_pairs{false}; + + // Whether out-of-order term pairs receive an additional penalty. + bool ordered{false}; + + // true=multiplicative, false=additive + bool boost_multiplicative{true}; + }; + // search uint32_t n_candidate{0}; // data cell float query_prune_ratio{0}; float term_prune_ratio{0}; + + // proximity scoring + ProximityBoostParameter proximity_boost; + + // phrase filter + std::vector phrase_terms; + uint32_t phrase_slop{0}; }; } // namespace vsag diff --git a/src/algorithm/sindi/sindi_parameter_test.cpp b/src/algorithm/sindi/sindi_parameter_test.cpp index 614beb495d..d74b332c1e 100644 --- a/src/algorithm/sindi/sindi_parameter_test.cpp +++ b/src/algorithm/sindi/sindi_parameter_test.cpp @@ -48,6 +48,8 @@ struct SINDIDefaultParam { int avg_doc_term_length = 100; bool remap_term_ids = false; bool immutable = false; + bool store_positions = false; + int max_positions_per_term = 64; }; std::string @@ -66,6 +68,8 @@ generate_sindi_param(const SINDIDefaultParam& param) { json[SPARSE_AVG_DOC_TERM_LENGTH].SetInt(param.avg_doc_term_length); json[SPARSE_REMAP_TERM_IDS].SetBool(param.remap_term_ids); json[SPARSE_IMMUTABLE].SetBool(param.immutable); + json[SPARSE_STORE_POSITIONS].SetBool(param.store_positions); + json[SPARSE_MAX_POSITIONS_PER_TERM].SetInt(param.max_positions_per_term); return json.Dump(); } @@ -84,6 +88,9 @@ TEST_CASE("SINDI Index Parameters Test", "[ut][SINDIParameter]") { REQUIRE(param->avg_doc_term_length == default_param.avg_doc_term_length); REQUIRE(param->remap_term_ids == default_param.remap_term_ids); REQUIRE(param->immutable == default_param.immutable); + REQUIRE(param->store_positions == default_param.store_positions); + REQUIRE(param->max_positions_per_term == + static_cast(default_param.max_positions_per_term)); vsag::ParameterTest::TestToJson(param); REQUIRE_FALSE(param->ToJson().Contains(SPARSE_IMMUTABLE)); @@ -92,14 +99,48 @@ TEST_CASE("SINDI Index Parameters Test", "[ut][SINDIParameter]") { "sindi": { "query_prune_ratio": 0.2, "n_candidate": 20, - "term_prune_ratio": 0.1 + "term_prune_ratio": 0.1, + "proximity_boost": { + "candidates": 5000, + "weight": 0.3, + "all_pairs": true, + "ordered": true, + "boost_multiplicative": false + }, + "phrase_terms": [12, 34, 56], + "phrase_slop": 1 } })"; auto search_param = std::make_shared(); vsag::JsonType search_param_json = vsag::JsonType::Parse(search_param_str); search_param->FromJson(search_param_json); + REQUIRE(search_param->proximity_boost.candidates == 5000); + REQUIRE(std::abs(search_param->proximity_boost.weight - 0.3f) < 1e-6f); + REQUIRE(search_param->proximity_boost.all_pairs); + REQUIRE(search_param->proximity_boost.ordered == true); + REQUIRE(search_param->proximity_boost.boost_multiplicative == false); + const std::vector expected_phrase_terms{12, 34, 56}; + REQUIRE(search_param->phrase_terms == expected_phrase_terms); + REQUIRE(search_param->phrase_slop == 1); + vsag::ParameterTest::TestToJson(search_param); - REQUIRE_FALSE(search_param->ToJson()[INDEX_SINDI].Contains("use_term_lists_heap_insert")); + auto serialized = search_param->ToJson(); + auto serialized_sindi = serialized[INDEX_SINDI]; + REQUIRE_FALSE(serialized_sindi.Contains("use_term_lists_heap_insert")); + REQUIRE_FALSE(serialized_sindi.Contains("proximity_weight")); + + auto serialized_proximity = serialized_sindi[SPARSE_PROXIMITY_BOOST]; + REQUIRE(serialized_proximity.IsObject()); + REQUIRE(serialized_proximity[SPARSE_PROXIMITY_BOOST_CANDIDATES].GetInt() == 5000); + REQUIRE(std::abs(serialized_proximity[SPARSE_PROXIMITY_BOOST_WEIGHT].GetFloat() - 0.3f) < + 1e-6f); + REQUIRE(serialized_proximity[SPARSE_PROXIMITY_BOOST_ALL_PAIRS].GetBool()); + REQUIRE(serialized_proximity[SPARSE_PROXIMITY_BOOST_ORDERED].GetBool()); + REQUIRE_FALSE(serialized_proximity[SPARSE_PROXIMITY_BOOST_MULTIPLICATIVE].GetBool()); + const std::vector expected_serialized_phrase_terms{12, 34, 56}; + REQUIRE(serialized_sindi[SPARSE_PHRASE_TERMS].GetVector() == + expected_serialized_phrase_terms); + REQUIRE(serialized_sindi[SPARSE_PHRASE_SLOP].GetInt() == 1); auto legacy_search_param_str = R"({ "sindi": { @@ -115,6 +156,28 @@ TEST_CASE("SINDI Index Parameters Test", "[ut][SINDIParameter]") { legacy_search_param->ToJson()[INDEX_SINDI].Contains("use_term_lists_heap_insert")); } +TEST_CASE("SINDI proximity boost parameters", "[ut][SINDIParameter]") { + SECTION("defaults preserve disabled proximity scoring") { + auto param = std::make_shared(); + param->FromString(R"({"sindi": {}})"); + + REQUIRE(std::abs(param->proximity_boost.weight) < 1e-6f); + REQUIRE(param->proximity_boost.candidates == 10000); + REQUIRE_FALSE(param->proximity_boost.all_pairs); + REQUIRE_FALSE(param->proximity_boost.ordered); + REQUIRE(param->proximity_boost.boost_multiplicative); + } + + SECTION("requires an object and a positive candidate count") { + auto param = std::make_shared(); + REQUIRE_THROWS_AS(param->FromString(R"({"sindi": {"proximity_boost": 0.3}})"), + vsag::VsagException); + REQUIRE_THROWS_AS( + param->FromString(R"({"sindi": {"proximity_boost": {"candidates": 0}}})"), + vsag::VsagException); + } +} + TEST_CASE("SINDI Index Parameters Compatibility Test", "[ut][SINDIParameter]") { TEST_COMPATIBILITY_CASE("use_reorder compatibility", use_reorder, true, false, false); TEST_COMPATIBILITY_CASE("value quantization compatibility", @@ -129,6 +192,9 @@ TEST_CASE("SINDI Index Parameters Compatibility Test", "[ut][SINDIParameter]") { "avg_doc_term_length compatibility", avg_doc_term_length, 100, 200, false); TEST_COMPATIBILITY_CASE("remap_term_ids compatibility", remap_term_ids, false, true, false); TEST_COMPATIBILITY_CASE("immutable compatibility", immutable, false, true, false); + TEST_COMPATIBILITY_CASE("store_positions compatibility", store_positions, false, true, false); + TEST_COMPATIBILITY_CASE( + "max_positions_per_term compatibility", max_positions_per_term, 64, 32, false); } TEST_CASE("SINDI immutable Parameter", "[ut][SINDIParameter]") { @@ -238,3 +304,39 @@ TEST_CASE("SINDI remap_term_ids Parameter", "[ut][SINDIParameter]") { REQUIRE(param2->remap_term_ids == true); } } + +TEST_CASE("SINDI store_positions Parameter", "[ut][SINDIParameter]") { + SECTION("default is false when not specified") { + auto param_str = R"({"term_id_limit": 1000, "window_size": 50000})"; + auto param = std::make_shared(); + param->FromJson(vsag::JsonType::Parse(param_str)); + REQUIRE(param->store_positions == false); + REQUIRE(param->max_positions_per_term == 64); + } + + SECTION("parse store_positions true") { + SINDIDefaultParam dp; + dp.store_positions = true; + dp.max_positions_per_term = 32; + auto param_str = generate_sindi_param(dp); + auto param = std::make_shared(); + param->FromString(param_str); + REQUIRE(param->store_positions == true); + REQUIRE(param->max_positions_per_term == 32); + } + + SECTION("round-trip: FromJson -> ToJson -> FromJson") { + SINDIDefaultParam dp; + dp.store_positions = true; + dp.max_positions_per_term = 128; + auto param_str = generate_sindi_param(dp); + auto param1 = std::make_shared(); + param1->FromString(param_str); + + auto json2 = param1->ToJson(); + auto param2 = std::make_shared(); + param2->FromJson(json2); + REQUIRE(param2->store_positions == true); + REQUIRE(param2->max_positions_per_term == 128); + } +} diff --git a/src/algorithm/sindi/sindi_test.cpp b/src/algorithm/sindi/sindi_test.cpp index 152472179f..145b6f099c 100644 --- a/src/algorithm/sindi/sindi_test.cpp +++ b/src/algorithm/sindi/sindi_test.cpp @@ -17,11 +17,14 @@ #include "sindi.h" #include +#include #include #include #include #include +#include +#include "algorithm/sindi/proximity_scorer.h" #include "impl/allocator/safe_allocator.h" #include "index_common_param.h" #include "storage/serialization_tags.h" @@ -1885,3 +1888,1887 @@ TEST_CASE("SINDI Remap Memory Comparison - MD5 Vocabulary", "[ut][SINDI]") { delete[] item.ids_; } } + +TEST_CASE("SINDI Proximity Basic Test", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Create controlled data: 3 docs, same terms, different positions + // Doc 0: terms [10, 20, 30] clustered at positions [0, 1, 2] → high proximity + // Doc 1: terms [10, 20, 30] scattered at positions [0, 50, 100] → low proximity + // Doc 2: terms [10, 20] at positions [0, 1] → partial match, high proximity + uint32_t num_base = 3; + + SparseVector sv_base_prox[3]; + + uint32_t ids0[] = {10, 20, 30}; + float vals0[] = {0.3f, 0.3f, 0.3f}; + uint32_t seq0[] = {10, 20, 30}; + sv_base_prox[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {10, 20, 30}; + float vals1[] = {0.3f, 0.3f, 0.3f}; + std::vector seq1_vec(101, 99); + seq1_vec[0] = 10; + seq1_vec[50] = 20; + seq1_vec[100] = 30; + sv_base_prox[1] = {3, ids1, vals1, 101, seq1_vec.data()}; + + uint32_t ids2[] = {10, 20}; + float vals2[] = {0.3f, 0.3f}; + uint32_t seq2[] = {10, 20}; + sv_base_prox[2] = {2, ids2, vals2, 2, seq2}; + + std::vector base_ids = {0, 1, 2}; + auto base = vsag::Dataset::Make(); + base->NumElements(num_base)->SparseVectors(sv_base_prox)->Ids(base_ids.data())->Owner(false); + + auto param_str_prox = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType param_json = vsag::JsonType::Parse(param_str_prox); + auto index_param = std::make_shared(); + index_param->FromJson(param_json); + auto index = std::make_unique(index_param, common_param); + + auto build_res = index->Build(base); + REQUIRE(build_res.size() == 0); + REQUIRE(index->GetNumElements() == 3); + + // Query: terms [10, 20, 30] + uint32_t q_ids[] = {10, 20, 30}; + float q_vals[] = {0.3f, 0.3f, 0.3f}; + SparseVector query_sv = {3, q_ids, q_vals, 0, nullptr}; + auto query_ds = vsag::Dataset::Make(); + query_ds->NumElements(1)->SparseVectors(&query_sv)->Owner(false); + + // Search WITHOUT proximity boost + std::string search_no_prox = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.0 + } + } + })"; + auto result_no_prox = index->KnnSearch(query_ds, 3, search_no_prox, nullptr); + REQUIRE(result_no_prox->GetDim() == 3); + // Doc 0 and Doc 1 have same IP score (both have terms 10,20,30 with weight 0.3) + // IP = 0.3*0.3 * 3 = 0.27, distance = 1 - 0.27 = 0.73 + REQUIRE(result_no_prox->GetDistances()[0] == result_no_prox->GetDistances()[1]); + + // Search WITH proximity boost + std::string search_with_prox = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.5, + "ordered": false, + "candidates": 10000 + } + } + })"; + auto result_prox = index->KnnSearch(query_ds, 3, search_with_prox, nullptr); + REQUIRE(result_prox->GetDim() == 3); + // Doc 0 (clustered) should rank before Doc 1 (scattered) + REQUIRE(result_prox->GetIds()[0] == 0); + REQUIRE(result_prox->GetDistances()[0] < result_prox->GetDistances()[1]); +} + +TEST_CASE("SINDI Proximity Disabled Regression", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + uint32_t num_base = 100; + int64_t max_dim = 32; + int64_t max_id = 500; + int64_t k = 10; + + std::vector ids(num_base); + for (int64_t i = 0; i < num_base; ++i) { + ids[i] = i; + } + + auto sv_base_reg = fixtures::GenerateSparseVectors(num_base, max_dim, max_id, 0, 10, 42); + + // Add token sequences + std::mt19937 rng(42); + std::vector> token_seqs(num_base); + for (uint32_t i = 0; i < num_base; ++i) { + uint32_t seq_len = 50 + rng() % 100; + token_seqs[i].resize(seq_len); + for (uint32_t j = 0; j < seq_len; ++j) { + token_seqs[i][j] = rng() % max_id; + } + sv_base_reg[i].token_seq_len_ = seq_len; + sv_base_reg[i].token_sequence_ = token_seqs[i].data(); + } + + auto base_reg = vsag::Dataset::Make(); + base_reg->NumElements(num_base) + ->SparseVectors(sv_base_reg.data()) + ->Ids(ids.data()) + ->Owner(false); + + auto param_str_reg = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 501, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 32 + })"; + + vsag::JsonType param_json_reg = vsag::JsonType::Parse(param_str_reg); + auto index_param_reg = std::make_shared(); + index_param_reg->FromJson(param_json_reg); + auto index_reg = std::make_unique(index_param_reg, common_param); + auto build_res_reg = index_reg->Build(base_reg); + REQUIRE(build_res_reg.size() == 0); + + // proximity_weight=0 should be deterministic and consistent + std::string search_no_prox_reg = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 20, + "proximity_boost": { + "weight": 0.0 + } + } + })"; + + auto query_reg = vsag::Dataset::Make(); + for (int i = 0; i < 10; ++i) { + query_reg->NumElements(1)->SparseVectors(sv_base_reg.data() + i)->Owner(false); + auto result1 = index_reg->KnnSearch(query_reg, k, search_no_prox_reg, nullptr); + auto result2 = index_reg->KnnSearch(query_reg, k, search_no_prox_reg, nullptr); + REQUIRE(result1->GetDim() == result2->GetDim()); + for (int j = 0; j < result1->GetDim(); ++j) { + REQUIRE(result1->GetIds()[j] == result2->GetIds()[j]); + } + } + + for (auto& item : sv_base_reg) { + delete[] item.vals_; + delete[] item.ids_; + } +} + +TEST_CASE("SINDI Proximity Score Verification", "[ut][SINDI][Proximity]") { + // Verify exact scores for both multiplicative and additive modes + // Use small weights so distance falls in [0, 1] + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // 2 docs, both contain [10, 20, 30] with weight 0.3 + // Doc 0: clustered at [0, 1, 2] + // Doc 1: scattered at [0, 50, 100] + SparseVector sv_docs[2]; + uint32_t ids0[] = {10, 20, 30}; + float vals0[] = {0.3f, 0.3f, 0.3f}; + uint32_t seq0[] = {10, 20, 30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {10, 20, 30}; + float vals1[] = {0.3f, 0.3f, 0.3f}; + std::vector seq1_vec(101, 99); + seq1_vec[0] = 10; + seq1_vec[50] = 20; + seq1_vec[100] = 30; + sv_docs[1] = {3, ids1, vals1, 101, seq1_vec.data()}; + + std::vector base_ids = {0, 1}; + auto base = vsag::Dataset::Make(); + base->NumElements(2)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + index->Build(base); + + uint32_t q_ids[] = {10, 20, 30}; + float q_vals[] = {0.3f, 0.3f, 0.3f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + // IP score for both docs: 0.3*0.3 + 0.3*0.3 + 0.3*0.3 = 0.27 + // Without proximity: distance = 1 - 0.27 = 0.73 + float ip_score = 0.27f; + + // All-pairs proximity (proximity_all_pairs=true): score all C(3,2)=3 pairs. + // Doc 0: (10,20) dist=1→1/2, (10,30) dist=2→1/3, (20,30) dist=1→1/2 + float raw_boost_0_ap = 1.0f / 2.0f + 1.0f / 3.0f + 1.0f / 2.0f; + // Doc 1: (10,20) dist=50→1/51, (10,30) dist=100→1/101, (20,30) dist=50→1/51 + float raw_boost_1_ap = 1.0f / 51.0f + 1.0f / 101.0f + 1.0f / 51.0f; + float pair_count_ap = 3.0f; // C(3,2) + float norm_boost_0_ap = raw_boost_0_ap / pair_count_ap; + float norm_boost_1_ap = raw_boost_1_ap / pair_count_ap; + + // Adjacent-only proximity (proximity_all_pairs=false, the default): score + // only query-adjacent pairs (10,20) and (20,30) → n-1=2 pairs. + float raw_boost_0_adj = 1.0f / 2.0f + 1.0f / 2.0f; + float raw_boost_1_adj = 1.0f / 51.0f + 1.0f / 51.0f; + float pair_count_adj = 2.0f; // n-1 + float norm_boost_0_adj = raw_boost_0_adj / pair_count_adj; + float norm_boost_1_adj = raw_boost_1_adj / pair_count_adj; + + float beta = 0.3f; + + SECTION("multiplicative mode, all-pairs") { + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.3, + "boost_multiplicative": true, + "all_pairs": true + } + } + })"; + auto result = index->KnnSearch(query, 2, sp, nullptr); + // dists[doc] = -0.27 (negative IP) + // multiplicative: dists *= (1 + beta * norm_boost) + // final_distance = 1 + dists_after + float expected_0 = 1.0f + (-ip_score) * (1.0f + beta * norm_boost_0_ap); + float expected_1 = 1.0f + (-ip_score) * (1.0f + beta * norm_boost_1_ap); + REQUIRE(result->GetIds()[0] == 0); + REQUIRE(result->GetDistances()[0] > 0.0f); // distance in [0,1] + REQUIRE(result->GetDistances()[0] < 1.0f); + REQUIRE(result->GetDistances()[1] > 0.0f); + REQUIRE(result->GetDistances()[1] < 1.0f); + REQUIRE(std::abs(result->GetDistances()[0] - expected_0) < 1e-4f); + REQUIRE(std::abs(result->GetDistances()[1] - expected_1) < 1e-4f); + } + + SECTION("additive mode, all-pairs") { + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.3, + "boost_multiplicative": false, + "all_pairs": true + } + } + })"; + auto result = index->KnnSearch(query, 2, sp, nullptr); + // additive: dists -= beta * norm_boost + // final_distance = 1 + (dists - beta * norm_boost) = 1 - ip - beta * norm_boost + float expected_0 = 1.0f - ip_score - beta * norm_boost_0_ap; + float expected_1 = 1.0f - ip_score - beta * norm_boost_1_ap; + REQUIRE(result->GetIds()[0] == 0); + REQUIRE(result->GetDistances()[0] > 0.0f); + REQUIRE(result->GetDistances()[0] < 1.0f); + REQUIRE(result->GetDistances()[1] > 0.0f); + REQUIRE(result->GetDistances()[1] < 1.0f); + REQUIRE(std::abs(result->GetDistances()[0] - expected_0) < 1e-4f); + REQUIRE(std::abs(result->GetDistances()[1] - expected_1) < 1e-4f); + } + + SECTION("multiplicative mode, adjacent-only (default)") { + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.3, + "boost_multiplicative": true + } + } + })"; + auto result = index->KnnSearch(query, 2, sp, nullptr); + float expected_0 = 1.0f + (-ip_score) * (1.0f + beta * norm_boost_0_adj); + float expected_1 = 1.0f + (-ip_score) * (1.0f + beta * norm_boost_1_adj); + REQUIRE(result->GetIds()[0] == 0); + REQUIRE(result->GetDistances()[0] > 0.0f); + REQUIRE(result->GetDistances()[0] < 1.0f); + REQUIRE(result->GetDistances()[1] > 0.0f); + REQUIRE(result->GetDistances()[1] < 1.0f); + REQUIRE(std::abs(result->GetDistances()[0] - expected_0) < 1e-4f); + REQUIRE(std::abs(result->GetDistances()[1] - expected_1) < 1e-4f); + } + + SECTION("additive mode, adjacent-only (default)") { + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.3, + "boost_multiplicative": false + } + } + })"; + auto result = index->KnnSearch(query, 2, sp, nullptr); + float expected_0 = 1.0f - ip_score - beta * norm_boost_0_adj; + float expected_1 = 1.0f - ip_score - beta * norm_boost_1_adj; + REQUIRE(result->GetIds()[0] == 0); + REQUIRE(result->GetDistances()[0] > 0.0f); + REQUIRE(result->GetDistances()[0] < 1.0f); + REQUIRE(result->GetDistances()[1] > 0.0f); + REQUIRE(result->GetDistances()[1] < 1.0f); + REQUIRE(std::abs(result->GetDistances()[0] - expected_0) < 1e-4f); + REQUIRE(std::abs(result->GetDistances()[1] - expected_1) < 1e-4f); + } +} + +TEST_CASE("SINDI Proximity Additive Mode", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Same setup as basic test with small weights + SparseVector sv_docs[2]; + uint32_t ids0[] = {10, 20, 30}; + float vals0[] = {0.3f, 0.3f, 0.3f}; + uint32_t seq0[] = {10, 20, 30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {10, 20, 30}; + float vals1[] = {0.3f, 0.3f, 0.3f}; + std::vector seq1_vec(101, 99); + seq1_vec[0] = 10; + seq1_vec[50] = 20; + seq1_vec[100] = 30; + sv_docs[1] = {3, ids1, vals1, 101, seq1_vec.data()}; + + std::vector base_ids = {0, 1}; + auto base = vsag::Dataset::Make(); + base->NumElements(2)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip_param = std::make_shared(); + ip_param->FromJson(pj); + auto index = std::make_unique(ip_param, common_param); + index->Build(base); + + uint32_t q_ids[] = {10, 20, 30}; + float q_vals[] = {0.3f, 0.3f, 0.3f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + // Additive mode + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.3, + "boost_multiplicative": false, + "candidates": 10000 + } + } + })"; + auto result = index->KnnSearch(query, 2, sp, nullptr); + REQUIRE(result->GetIds()[0] == 0); + REQUIRE(result->GetDistances()[0] < result->GetDistances()[1]); + // Distances should be in [0, 1] + REQUIRE(result->GetDistances()[0] > 0.0f); + REQUIRE(result->GetDistances()[0] < 1.0f); + REQUIRE(result->GetDistances()[1] > 0.0f); + REQUIRE(result->GetDistances()[1] < 1.0f); +} + +TEST_CASE("SINDI Proximity Serialize Round-Trip", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + SparseVector sv_docs[2]; + uint32_t ids0[] = {10, 20, 30}; + float vals0[] = {1.0f, 1.0f, 1.0f}; + uint32_t seq0[] = {10, 20, 30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {10, 20, 30}; + float vals1[] = {1.0f, 1.0f, 1.0f}; + std::vector seq1_vec(101, 99); + seq1_vec[0] = 10; + seq1_vec[50] = 20; + seq1_vec[100] = 30; + sv_docs[1] = {3, ids1, vals1, 101, seq1_vec.data()}; + + std::vector base_ids = {0, 1}; + auto base = vsag::Dataset::Make(); + base->NumElements(2)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip1 = std::make_shared(); + ip1->FromJson(pj); + auto index1 = std::make_unique(ip1, common_param); + index1->Build(base); + + // Search before serialize + uint32_t q_ids[] = {10, 20, 30}; + float q_vals[] = {1.0f, 1.0f, 1.0f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.5, + "boost_multiplicative": true + } + } + })"; + auto result_before = index1->KnnSearch(query, 2, sp, nullptr); + + // Serialize → Deserialize + auto ip2 = std::make_shared(); + ip2->FromJson(pj); + auto index2 = std::make_unique(ip2, common_param); + test_serializion(*index1, *index2); + + // Search after deserialize — results should match + auto result_after = index2->KnnSearch(query, 2, sp, nullptr); + REQUIRE(result_after->GetDim() == result_before->GetDim()); + for (int j = 0; j < result_after->GetDim(); ++j) { + REQUIRE(result_after->GetIds()[j] == result_before->GetIds()[j]); + REQUIRE(std::abs(result_after->GetDistances()[j] - result_before->GetDistances()[j]) < + 1e-6f); + } +} + +TEST_CASE("SINDI Proximity Streaming Serialize Round-Trip", + "[ut][SINDI][Proximity][PhraseFilter][streaming]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + common_param.metric_ = MetricType::METRIC_TYPE_IP; + + const bool remap_term_ids = GENERATE(false, true); + constexpr uint32_t term_offset_remap = 5000000; + const uint32_t term_offset = remap_term_ids ? term_offset_remap : 0; + const uint32_t t10 = term_offset + 10; + const uint32_t t20 = term_offset + 20; + const uint32_t t30 = term_offset + 30; + + SparseVector sv_docs[3]; + uint32_t ids0[] = {t10, t20, t30}; + float vals0[] = {1.0f, 1.0f, 1.0f}; + uint32_t seq0[] = {t10, t20, t30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {t10, t20, t30}; + float vals1[] = {0.7f, 0.7f, 0.7f}; + std::vector seq1_vec(101, term_offset + 99); + seq1_vec[0] = t10; + seq1_vec[50] = t20; + seq1_vec[100] = t30; + sv_docs[1] = {3, ids1, vals1, 101, seq1_vec.data()}; + + uint32_t ids2[] = {t10, t20}; + float vals2[] = {0.8f, 0.8f}; + uint32_t seq2[] = {t10, t20}; + sv_docs[2] = {2, ids2, vals2, 2, seq2}; + + std::vector base_ids = {0, 1, 2}; + auto base = Dataset::Make(); + base->NumElements(3)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = fmt::format(R"({{ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "remap_term_ids": {}, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + }})", + remap_term_ids); + auto param_json = JsonType::Parse(param_str); + auto param = std::make_shared(); + param->FromJson(param_json); + + auto index = std::make_unique(param, common_param); + REQUIRE(index->Build(base).empty()); + + uint32_t q_ids[] = {t10, t20, t30}; + float q_vals[] = {1.0f, 1.0f, 1.0f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + auto search_param = fmt::format(R"({{ + "sindi": {{ + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": {{ + "weight": 0.5, + "candidates": 10000 + }}, + "phrase_terms": [{}, {}, {}], + "phrase_slop": 5 + }} + }})", + t10, + t20, + t30); + auto result_before = index->KnnSearch(query, 3, search_param, nullptr); + + std::stringstream stream; + REQUIRE_NOTHROW(index->SerializeStreaming(stream)); + + auto restored = std::make_unique(param, common_param); + std::stringstream deserialize_stream(stream.str()); + REQUIRE_NOTHROW(restored->DeserializeStreaming(deserialize_stream)); + + auto result_after = restored->KnnSearch(query, 3, search_param, nullptr); + REQUIRE(result_after->GetDim() == result_before->GetDim()); + for (int64_t j = 0; j < result_after->GetDim(); ++j) { + REQUIRE(result_after->GetIds()[j] == result_before->GetIds()[j]); + REQUIRE(std::abs(result_after->GetDistances()[j] - result_before->GetDistances()[j]) < + 1e-6f); + } +} + +TEST_CASE("SINDI Proximity with Remap", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Use large term IDs to trigger remap + constexpr uint32_t offset = 2000000; + + SparseVector sv_docs[2]; + uint32_t ids0[] = {offset + 10, offset + 20, offset + 30}; + float vals0[] = {1.0f, 1.0f, 1.0f}; + uint32_t seq0[] = {offset + 10, offset + 20, offset + 30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {offset + 10, offset + 20, offset + 30}; + float vals1[] = {1.0f, 1.0f, 1.0f}; + std::vector seq1_vec(101, offset + 99); + seq1_vec[0] = offset + 10; + seq1_vec[50] = offset + 20; + seq1_vec[100] = offset + 30; + sv_docs[1] = {3, ids1, vals1, 101, seq1_vec.data()}; + + std::vector base_ids = {0, 1}; + auto base = vsag::Dataset::Make(); + base->NumElements(2)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 100, + "remap_term_ids": true, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + auto build_res = index->Build(base); + REQUIRE(build_res.size() == 0); + + // Query with original (large) term IDs + uint32_t q_ids[] = {offset + 10, offset + 20, offset + 30}; + float q_vals[] = {1.0f, 1.0f, 1.0f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.5, + "boost_multiplicative": true + } + } + })"; + auto result = index->KnnSearch(query, 2, sp, nullptr); + REQUIRE(result->GetDim() == 2); + // Doc 0 (clustered) should still rank before Doc 1 (scattered) even with remap + REQUIRE(result->GetIds()[0] == 0); + REQUIRE(result->GetDistances()[0] < result->GetDistances()[1]); +} + +TEST_CASE("SINDI Proximity with Reorder", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + common_param.metric_ = MetricType::METRIC_TYPE_IP; + + SparseVector sv_docs[2]; + uint32_t ids0[] = {10, 20, 30}; + float vals0[] = {1.0f, 1.0f, 1.0f}; + uint32_t seq0[] = {10, 20, 30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {10, 20, 30}; + float vals1[] = {1.0f, 1.0f, 1.0f}; + std::vector seq1_vec(101, 99); + seq1_vec[0] = 10; + seq1_vec[50] = 20; + seq1_vec[100] = 30; + sv_docs[1] = {3, ids1, vals1, 101, seq1_vec.data()}; + + std::vector base_ids = {0, 1}; + auto base = vsag::Dataset::Make(); + base->NumElements(2)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": true, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + index->Build(base); + + uint32_t q_ids[] = {10, 20, 30}; + float q_vals[] = {1.0f, 1.0f, 1.0f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + // With reorder + proximity, both should work together + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.5, + "boost_multiplicative": true + } + } + })"; + auto result = index->KnnSearch(query, 2, sp, nullptr); + REQUIRE(result->GetDim() == 2); + // Doc 0 (clustered) should rank first + REQUIRE(result->GetIds()[0] == 0); +} + +TEST_CASE("SINDI Proximity with DocPrune", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Doc with many terms, doc_prune will remove low-weight ones + SparseVector sv_docs[2]; + uint32_t ids0[] = {10, 20, 30, 40, 50}; + float vals0[] = {5.0f, 4.0f, 0.1f, 0.1f, 0.1f}; // 30,40,50 will be pruned + uint32_t seq0[] = {10, 20, 30, 40, 50}; + sv_docs[0] = {5, ids0, vals0, 5, seq0}; + + uint32_t ids1[] = {10, 20, 30, 40, 50}; + float vals1[] = {5.0f, 4.0f, 0.1f, 0.1f, 0.1f}; + std::vector seq1_vec(101, 99); + seq1_vec[0] = 10; + seq1_vec[50] = 20; + seq1_vec[60] = 30; + seq1_vec[70] = 40; + seq1_vec[80] = 50; + sv_docs[1] = {5, ids1, vals1, 101, seq1_vec.data()}; + + std::vector base_ids = {0, 1}; + auto base = vsag::Dataset::Make(); + base->NumElements(2)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + // doc_prune_ratio=0.5 will prune ~50% of term mass + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.5, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + auto build_res = index->Build(base); + REQUIRE(build_res.size() == 0); + + // Should not crash — pruned terms just have no positions + uint32_t q_ids[] = {10, 20}; + float q_vals[] = {1.0f, 1.0f}; + SparseVector qsv = {2, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.5 + } + } + })"; + auto result = index->KnnSearch(query, 2, sp, nullptr); + REQUIRE(result->GetDim() == 2); +} + +TEST_CASE("SINDI Proximity with Filter", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + SparseVector sv_docs[3]; + uint32_t ids0[] = {10, 20, 30}; + float vals0[] = {1.0f, 1.0f, 1.0f}; + uint32_t seq0[] = {10, 20, 30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; // id=0 even → pass filter + + uint32_t ids1[] = {10, 20, 30}; + float vals1[] = {1.0f, 1.0f, 1.0f}; + uint32_t seq1[] = {10, 20, 30}; + sv_docs[1] = {3, ids1, vals1, 3, seq1}; // id=1 odd → filtered out + + uint32_t ids2[] = {10, 20, 30}; + float vals2[] = {1.0f, 1.0f, 1.0f}; + std::vector seq2_vec(101, 99); + seq2_vec[0] = 10; + seq2_vec[50] = 20; + seq2_vec[100] = 30; + sv_docs[2] = {3, ids2, vals2, 101, seq2_vec.data()}; // id=2 even → pass filter + + std::vector base_ids = {0, 1, 2}; + auto base = vsag::Dataset::Make(); + base->NumElements(3)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + index->Build(base); + + uint32_t q_ids[] = {10, 20, 30}; + float q_vals[] = {1.0f, 1.0f, 1.0f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + auto mock_filter = std::make_shared(); // even IDs only + + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.5 + } + } + })"; + auto result = index->KnnSearch(query, 3, sp, mock_filter); + // Only doc 0 and doc 2 should be returned (even IDs) + REQUIRE(result->GetDim() == 2); + // Doc 0 (clustered, id=0) should rank before Doc 2 (scattered, id=2) + REQUIRE(result->GetIds()[0] == 0); + REQUIRE(result->GetIds()[1] == 2); +} + +TEST_CASE("SINDI Proximity Multi-Window", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Use small window_size to force multiple windows + uint32_t num_base = 200; + uint32_t window_size = 10000; // min allowed + + std::vector ids(num_base); + std::vector sv_base(num_base); + std::vector> all_ids(num_base); + std::vector> all_vals(num_base); + std::vector> all_seqs(num_base); + + std::mt19937 rng(123); + for (uint32_t i = 0; i < num_base; ++i) { + ids[i] = i; + uint32_t n_terms = 5 + rng() % 10; + all_ids[i].resize(n_terms); + all_vals[i].resize(n_terms); + all_seqs[i].resize(20 + rng() % 30); + for (uint32_t j = 0; j < n_terms; ++j) { + all_ids[i][j] = rng() % 100; + all_vals[i][j] = 1.0f + static_cast(rng() % 10); + } + // Sort and dedup ids + std::sort(all_ids[i].begin(), all_ids[i].end()); + all_ids[i].erase(std::unique(all_ids[i].begin(), all_ids[i].end()), all_ids[i].end()); + all_vals[i].resize(all_ids[i].size()); + n_terms = all_ids[i].size(); + + for (uint32_t j = 0; j < all_seqs[i].size(); ++j) { + all_seqs[i][j] = rng() % 100; + } + + sv_base[i].len_ = n_terms; + sv_base[i].ids_ = all_ids[i].data(); + sv_base[i].vals_ = all_vals[i].data(); + sv_base[i].token_seq_len_ = all_seqs[i].size(); + sv_base[i].token_sequence_ = all_seqs[i].data(); + } + + auto base = vsag::Dataset::Make(); + base->NumElements(num_base)->SparseVectors(sv_base.data())->Ids(ids.data())->Owner(false); + + auto param_str = fmt::format(R"({{ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": {}, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + }})", + window_size); + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + auto build_res = index->Build(base); + REQUIRE(build_res.size() == 0); + + // Search with proximity — should not crash across windows + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(sv_base.data())->Owner(false); + + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 20, + "proximity_boost": { + "weight": 0.3 + } + } + })"; + auto result = index->KnnSearch(query, 10, sp, nullptr); + REQUIRE(result->GetDim() > 0); + + // Also verify proximity_weight=0 gives results + std::string sp_no = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 20, + "proximity_boost": { + "weight": 0.0 + } + } + })"; + auto result_no = index->KnnSearch(query, 10, sp_no, nullptr); + REQUIRE(result_no->GetDim() > 0); +} + +TEST_CASE("SINDI Proximity Ordered Integration", "[ut][SINDI][Proximity]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Doc 0: terms in forward order [10, 20] at positions [0, 1] + // Doc 1: terms in reverse order [10, 20] but 20 at pos 0, 10 at pos 1 + SparseVector sv_docs[2]; + uint32_t ids0[] = {10, 20}; + float vals0[] = {1.0f, 1.0f}; + uint32_t seq0[] = {10, 20}; // forward: 10@0, 20@1 + sv_docs[0] = {2, ids0, vals0, 2, seq0}; + + uint32_t ids1[] = {10, 20}; + float vals1[] = {1.0f, 1.0f}; + uint32_t seq1[] = {20, 10}; // reverse: 20@0, 10@1 + sv_docs[1] = {2, ids1, vals1, 2, seq1}; + + std::vector base_ids = {0, 1}; + auto base = vsag::Dataset::Make(); + base->NumElements(2)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + index->Build(base); + + // Query [10, 20] — ordered mode: 10 should come before 20 + uint32_t q_ids[] = {10, 20}; + float q_vals[] = {1.0f, 1.0f}; + SparseVector qsv = {2, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + // Unordered: both docs have same dist=1, same boost + std::string sp_unordered = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.5, + "ordered": false + } + } + })"; + auto result_unordered = index->KnnSearch(query, 2, sp_unordered, nullptr); + REQUIRE(result_unordered->GetDistances()[0] == result_unordered->GetDistances()[1]); + + // Ordered: Doc 0 is forward (dist=1), Doc 1 is reverse (dist=1*2=2) + std::string sp_ordered = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.5, + "ordered": true + } + } + })"; + auto result_ordered = index->KnnSearch(query, 2, sp_ordered, nullptr); + // Doc 0 should rank first in ordered mode + REQUIRE(result_ordered->GetIds()[0] == 0); + REQUIRE(result_ordered->GetDistances()[0] < result_ordered->GetDistances()[1]); +} + +TEST_CASE("SINDI Phrase Filter Basic", "[ut][SINDI][PhraseFilter]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Doc 0: [10, 20, 30] clustered at [0, 1, 2] → phrase passes (span=2) + // Doc 1: [10, 20, 30] scattered at [0, 50, 100] → phrase fails (span=100) + // Doc 2: [10, 20] only → phrase fails (missing term 30) + SparseVector sv_docs[3]; + uint32_t ids0[] = {10, 20, 30}; + float vals0[] = {0.3f, 0.3f, 0.3f}; + uint32_t seq0[] = {10, 20, 30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {10, 20, 30}; + float vals1[] = {0.3f, 0.3f, 0.3f}; + std::vector seq1_vec(101, 99); + seq1_vec[0] = 10; + seq1_vec[50] = 20; + seq1_vec[100] = 30; + sv_docs[1] = {3, ids1, vals1, 101, seq1_vec.data()}; + + uint32_t ids2[] = {10, 20}; + float vals2[] = {0.3f, 0.3f}; + uint32_t seq2[] = {10, 20}; + sv_docs[2] = {2, ids2, vals2, 2, seq2}; + + std::vector base_ids = {0, 1, 2}; + auto base = vsag::Dataset::Make(); + base->NumElements(3)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + index->Build(base); + + uint32_t q_ids[] = {10, 20, 30}; + float q_vals[] = {0.3f, 0.3f, 0.3f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + // phrase_terms=[10,20,30], slop=5 → max_span = 5+3-1=7 + // Doc 0: span=2 ≤ 7 → pass + // Doc 1: span=100 > 7 → filtered + // Doc 2: missing term 30 → filtered + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.0 + }, + "phrase_terms": [10, 20, 30], + "phrase_slop": 5 + } + })"; + auto result = index->KnnSearch(query, 3, sp, nullptr); + // Only Doc 0 survives + REQUIRE(result->GetDim() == 1); + REQUIRE(result->GetIds()[0] == 0); +} + +TEST_CASE("SINDI Phrase Filter Retained-Prefix Bound Regression", "[ut][SINDI][PhraseFilter]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Regression for the LookupPositions binary-search bound (docs-first + // plan verification item): a candidate doc that lives in the *pruned tail* + // of a second phrase term must read as position-absent, exactly as the old + // reverse-offset map did. If LookupPositions searched the full posting + // instead of the retained prefix, doc 8 below would wrongly pass the phrase. + // + // Layout (base_id == insertion index, single window): + // docs 0..7 : term 20 only -> term 20 posting = [0..8], doc 8 last + // doc 8 : terms 10 & 20 adjacent -> in term 10's retained prefix, but + // in term 20's pruned tail at ratio 0.5 + // doc 9 : term 10 only + // Query phrase = [10, 20]. Only doc 8 owns both terms, so it is the sole + // phrase candidate. With no term pruning it passes; with term_prune_ratio + // 0.5 term 20's tail (incl. doc 8) is pruned, so doc 8 must be filtered. + const uint32_t num_base = 10; + std::vector sv_docs(num_base); + std::vector> ids_store(num_base); + std::vector> vals_store(num_base); + std::vector> seq_store(num_base); + + for (uint32_t i = 0; i < 8; ++i) { + ids_store[i] = {20}; + vals_store[i] = {0.3f}; + seq_store[i] = {20}; + sv_docs[i] = {1, ids_store[i].data(), vals_store[i].data(), 1, seq_store[i].data()}; + } + // doc 8: both terms adjacent at positions 0 and 1 + ids_store[8] = {10, 20}; + vals_store[8] = {0.3f, 0.3f}; + seq_store[8] = {10, 20}; + sv_docs[8] = {2, ids_store[8].data(), vals_store[8].data(), 2, seq_store[8].data()}; + // doc 9: term 10 only + ids_store[9] = {10}; + vals_store[9] = {0.3f}; + seq_store[9] = {10}; + sv_docs[9] = {1, ids_store[9].data(), vals_store[9].data(), 1, seq_store[9].data()}; + + std::vector base_ids(num_base); + for (uint32_t i = 0; i < num_base; ++i) { + base_ids[i] = i; + } + auto base = vsag::Dataset::Make(); + base->NumElements(num_base)->SparseVectors(sv_docs.data())->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip = std::make_shared(); + ip->FromJson(pj); + auto index = std::make_unique(ip, common_param); + index->Build(base); + + uint32_t q_ids[] = {10, 20}; + float q_vals[] = {0.3f, 0.3f}; + SparseVector qsv = {2, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + // No term pruning: full postings searched, doc 8 owns both terms adjacent + // -> phrase passes, doc 8 is returned. + std::string sp_no_prune = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 20, + "proximity_boost": { + "weight": 0.0 + }, + "phrase_terms": [10, 20], + "phrase_slop": 5 + } + })"; + auto res_full = index->KnnSearch(query, 10, sp_no_prune, nullptr); + bool doc8_present_full = false; + for (int64_t i = 0; i < res_full->GetDim(); ++i) { + if (res_full->GetIds()[i] == 8) { + doc8_present_full = true; + } + } + REQUIRE(doc8_present_full == true); + + // Term pruning 0.5: term 20's posting [0..8] retains only the head [0..3], + // so doc 8 (last) falls in the pruned tail and LookupPositions must + // treat term 20 as absent for doc 8 -> phrase fails -> doc 8 filtered out. + std::string sp_pruned = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.5, + "n_candidate": 20, + "proximity_boost": { + "weight": 0.0 + }, + "phrase_terms": [10, 20], + "phrase_slop": 5 + } + })"; + auto res_pruned = index->KnnSearch(query, 10, sp_pruned, nullptr); + bool doc8_present_pruned = false; + for (int64_t i = 0; i < res_pruned->GetDim(); ++i) { + if (res_pruned->GetIds()[i] == 8) { + doc8_present_pruned = true; + } + } + REQUIRE(doc8_present_pruned == false); +} + +TEST_CASE("SINDI Phrase Filter with Boost Combined", "[ut][SINDI][PhraseFilter]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // 3 docs all with terms [10, 20, 30] + // Doc 0: clustered [0,1,2] -> phrase pass + high boost + // Doc 1: medium [0,3,6] -> phrase pass (span=6, slop=5 -> max_span=7) + + // medium boost + // Doc 2: scattered [0,50,100] -> phrase fail (span=100 > 7) + SparseVector sv_docs[3]; + uint32_t ids0[] = {10, 20, 30}; + float vals0[] = {0.3f, 0.3f, 0.3f}; + uint32_t seq0[] = {10, 20, 30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + uint32_t ids1[] = {10, 20, 30}; + float vals1[] = {0.3f, 0.3f, 0.3f}; + uint32_t seq1[] = {10, 99, 99, 20, 99, 99, 30}; + sv_docs[1] = {3, ids1, vals1, 7, seq1}; + + uint32_t ids2[] = {10, 20, 30}; + float vals2[] = {0.3f, 0.3f, 0.3f}; + std::vector seq2_vec(101, 99); + seq2_vec[0] = 10; + seq2_vec[50] = 20; + seq2_vec[100] = 30; + sv_docs[2] = {3, ids2, vals2, 101, seq2_vec.data()}; + + std::vector base_ids = {0, 1, 2}; + auto base = vsag::Dataset::Make(); + base->NumElements(3)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto param_str = R"({ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10 + })"; + + vsag::JsonType pj = vsag::JsonType::Parse(param_str); + auto ip_param = std::make_shared(); + ip_param->FromJson(pj); + auto index = std::make_unique(ip_param, common_param); + index->Build(base); + + uint32_t q_ids[] = {10, 20, 30}; + float q_vals[] = {0.3f, 0.3f, 0.3f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + // phrase filter + boost combined + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 10, + "proximity_boost": { + "weight": 0.3 + }, + "phrase_terms": [10, 20, 30], + "phrase_slop": 5 + } + })"; + auto result = index->KnnSearch(query, 3, sp, nullptr); + // Doc 2 filtered out, only Doc 0 and Doc 1 remain + REQUIRE(result->GetDim() == 2); + // Doc 0 (more clustered) should rank before Doc 1 + REQUIRE(result->GetIds()[0] == 0); + REQUIRE(result->GetIds()[1] == 1); + REQUIRE(result->GetDistances()[0] < result->GetDistances()[1]); +} + +TEST_CASE("SINDI Immutable Proximity Phrase Equality", "[ut][SINDI][Proximity][PhraseFilter]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + common_param.metric_ = MetricType::METRIC_TYPE_IP; + + const char* sparse_value_quant_type = GENERATE("fp32", "sq8", "fp16"); + const bool remap_term_ids = GENERATE(false, true); + const std::string use_quantization = + std::string(sparse_value_quant_type) == "fp16" + ? R"("fp16")" + : (std::string(sparse_value_quant_type) == "sq8" ? "true" : "false"); + + // Controlled docs so both proximity ordering and phrase filtering exercise + // real positions. term_offset lets us optionally push term ids into the + // remap range. Every doc shares terms {t10,t20,t30} but at different spreads. + constexpr uint32_t term_offset_remap = 5000000; + const uint32_t term_offset = remap_term_ids ? term_offset_remap : 0; + const uint32_t t10 = term_offset + 10; + const uint32_t t20 = term_offset + 20; + const uint32_t t30 = term_offset + 30; + + constexpr uint32_t num_base = 4; + SparseVector sv_docs[num_base]; + + // Values are intentionally non-uniform: SQ8 quantization derives its + // min/max from the base set, so all-identical values would collapse every + // code to 0 and yield empty results. Distinct values keep the SQ8 path + // meaningful while phrase matching (position-only) is unaffected. + + // Doc 0: clustered [0,1,2] -> phrase pass, high proximity + uint32_t ids0[] = {t10, t20, t30}; + float vals0[] = {0.9f, 0.9f, 0.9f}; + uint32_t seq0[] = {t10, t20, t30}; + sv_docs[0] = {3, ids0, vals0, 3, seq0}; + + // Doc 1: medium [0,3,6] -> phrase pass (span 6 <= 7), medium proximity + uint32_t ids1[] = {t10, t20, t30}; + float vals1[] = {0.6f, 0.6f, 0.6f}; + uint32_t filler = term_offset + 99; + uint32_t seq1[] = {t10, filler, filler, t20, filler, filler, t30}; + sv_docs[1] = {3, ids1, vals1, 7, seq1}; + + // Doc 2: scattered [0,50,100] -> phrase fail, low proximity + uint32_t ids2[] = {t10, t20, t30}; + float vals2[] = {0.3f, 0.3f, 0.3f}; + std::vector seq2_vec(101, term_offset + 99); + seq2_vec[0] = t10; + seq2_vec[50] = t20; + seq2_vec[100] = t30; + sv_docs[2] = {3, ids2, vals2, 101, seq2_vec.data()}; + + // Doc 3: missing t30 -> phrase fail, partial proximity + uint32_t ids3[] = {t10, t20}; + float vals3[] = {0.5f, 0.5f}; + uint32_t seq3[] = {t10, t20}; + sv_docs[3] = {2, ids3, vals3, 2, seq3}; + + std::vector base_ids = {0, 1, 2, 3}; + auto base = vsag::Dataset::Make(); + base->NumElements(num_base)->SparseVectors(sv_docs)->Ids(base_ids.data())->Owner(false); + + auto make_param = [&](bool immutable) { + return fmt::format(R"({{ + "use_reorder": false, + "use_quantization": {}, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "remap_term_ids": {}, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10, + "immutable": {} + }})", + use_quantization, + remap_term_ids, + immutable); + }; + + auto source_param = std::make_shared(); + source_param->FromJson(vsag::JsonType::Parse(make_param(false))); + auto target_param = std::make_shared(); + target_param->FromJson(vsag::JsonType::Parse(make_param(true))); + + auto source = std::make_unique(source_param, common_param); + auto direct_immutable = std::make_unique(target_param, common_param); + auto immutable_roundtrip = std::make_unique(target_param, common_param); + + auto build_res = source->Build(base); + REQUIRE(build_res.empty()); + REQUIRE(direct_immutable->Build(base).empty()); + REQUIRE(direct_immutable->GetNumElements() == num_base); + test_serializion(*direct_immutable, *immutable_roundtrip); + REQUIRE(immutable_roundtrip->GetNumElements() == num_base); + + uint32_t q_ids[] = {t10, t20, t30}; + float q_vals[] = {0.3f, 0.3f, 0.3f}; + SparseVector qsv = {3, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + auto expect_equal = [](const DatasetPtr& a, const DatasetPtr& b) { + REQUIRE(a->GetDim() == b->GetDim()); + for (int64_t j = 0; j < a->GetDim(); ++j) { + REQUIRE(a->GetIds()[j] == b->GetIds()[j]); + REQUIRE(std::abs(a->GetDistances()[j] - b->GetDistances()[j]) < 1e-3); + } + }; + + // Proximity boost only. + std::string sp_prox = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 20, + "proximity_boost": { + "weight": 0.5, + "ordered": false, + "candidates": 10000 + } + } + })"; + auto source_prox_knn = source->KnnSearch(query, 4, sp_prox, nullptr); + expect_equal(source_prox_knn, direct_immutable->KnnSearch(query, 4, sp_prox, nullptr)); + expect_equal(source_prox_knn, immutable_roundtrip->KnnSearch(query, 4, sp_prox, nullptr)); + auto source_prox_range = source->RangeSearch(query, 1.0F, sp_prox, nullptr); + expect_equal(source_prox_range, direct_immutable->RangeSearch(query, 1.0F, sp_prox, nullptr)); + expect_equal(source_prox_range, + immutable_roundtrip->RangeSearch(query, 1.0F, sp_prox, nullptr)); + + // Phrase filter only. + std::string sp_phrase = fmt::format(R"({{ + "sindi": {{ + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 20, + "proximity_boost": {{ + "weight": 0.0 + }}, + "phrase_terms": [{}, {}, {}], + "phrase_slop": 5 + }} + }})", + t10, + t20, + t30); + auto src_phrase = source->KnnSearch(query, 4, sp_phrase, nullptr); + auto imm_phrase = direct_immutable->KnnSearch(query, 4, sp_phrase, nullptr); + expect_equal(src_phrase, imm_phrase); + expect_equal(src_phrase, immutable_roundtrip->KnnSearch(query, 4, sp_phrase, nullptr)); + // Only Doc 0 and Doc 1 pass the phrase constraint. + REQUIRE(imm_phrase->GetDim() == 2); + + // Phrase filter + proximity boost combined. + std::string sp_combined = fmt::format(R"({{ + "sindi": {{ + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 20, + "proximity_boost": {{ + "weight": 0.3 + }}, + "phrase_terms": [{}, {}, {}], + "phrase_slop": 5 + }} + }})", + t10, + t20, + t30); + auto source_combined = source->KnnSearch(query, 4, sp_combined, nullptr); + expect_equal(source_combined, direct_immutable->KnnSearch(query, 4, sp_combined, nullptr)); + expect_equal(source_combined, immutable_roundtrip->KnnSearch(query, 4, sp_combined, nullptr)); +} + +TEST_CASE("SINDI Immutable Proximity Ref Retained-Prefix Equality", "[ut][SINDI][Proximity]") { + // Regression for the pre-resolved posting-list view (ResolveTerm / + // LookupPositions): the resolved view must honor the same retained-prefix + // bound as the per-candidate lookup path. A candidate that lives in the + // pruned tail of one query term must read as position-absent, so its + // proximity contribution for that term is empty — identical to mutable. + // Exercised with term_prune_ratio > 0 and both remap settings. + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + common_param.metric_ = MetricType::METRIC_TYPE_IP; + + const bool remap_term_ids = GENERATE(false, true); + constexpr uint32_t term_offset_remap = 5000000; + const uint32_t term_offset = remap_term_ids ? term_offset_remap : 0; + const uint32_t t10 = term_offset + 10; + const uint32_t t20 = term_offset + 20; + + // Layout (single window, base_id == insertion index): + // docs 0..7 : term 10 & term 20, clustered so proximity is high + // doc 8 : term 10 & term 20 adjacent, but it is the LAST entry in each + // posting, so at term_prune_ratio 0.5 it falls in the pruned + // tail -> LookupPositions on both the mutable and immutable + // backends must skip it. + constexpr uint32_t num_base = 9; + std::vector sv_docs(num_base); + std::vector> ids_store(num_base); + std::vector> vals_store(num_base); + std::vector> seq_store(num_base); + + for (uint32_t i = 0; i < num_base; ++i) { + ids_store[i] = {t10, t20}; + // Descending values so the pruned tail (low IP) is the high doc ids. + float v = 0.9f - static_cast(i) * 0.08f; + vals_store[i] = {v, v}; + seq_store[i] = {t10, t20}; // adjacent -> non-zero proximity when present + sv_docs[i] = {2, ids_store[i].data(), vals_store[i].data(), 2, seq_store[i].data()}; + } + + std::vector base_ids(num_base); + for (uint32_t i = 0; i < num_base; ++i) { + base_ids[i] = i; + } + auto base = vsag::Dataset::Make(); + base->NumElements(num_base)->SparseVectors(sv_docs.data())->Ids(base_ids.data())->Owner(false); + + auto make_param = [&](bool immutable) { + return fmt::format(R"({{ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": 200, + "remap_term_ids": {}, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": 10, + "immutable": {} + }})", + remap_term_ids, + immutable); + }; + + auto source_param = std::make_shared(); + source_param->FromJson(vsag::JsonType::Parse(make_param(false))); + auto target_param = std::make_shared(); + target_param->FromJson(vsag::JsonType::Parse(make_param(true))); + + auto source = std::make_unique(source_param, common_param); + auto immutable = std::make_unique(target_param, common_param); + auto immutable_roundtrip = std::make_unique(target_param, common_param); + + REQUIRE(source->Build(base).empty()); + REQUIRE(immutable->Build(base).empty()); + REQUIRE(immutable->GetNumElements() == num_base); + test_serializion(*immutable, *immutable_roundtrip); + REQUIRE(immutable_roundtrip->GetNumElements() == num_base); + + uint32_t q_ids[] = {t10, t20}; + float q_vals[] = {0.5f, 0.5f}; + SparseVector qsv = {2, q_ids, q_vals, 0, nullptr}; + auto query = vsag::Dataset::Make(); + query->NumElements(1)->SparseVectors(&qsv)->Owner(false); + + auto expect_equal = [](const DatasetPtr& a, const DatasetPtr& b) { + REQUIRE(a->GetDim() == b->GetDim()); + for (int64_t j = 0; j < a->GetDim(); ++j) { + REQUIRE(a->GetIds()[j] == b->GetIds()[j]); + REQUIRE(std::abs(a->GetDistances()[j] - b->GetDistances()[j]) < 1e-3); + } + }; + + // term_prune_ratio 0.5 activates the retained-prefix bound in both backends. + std::string sp = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.5, + "n_candidate": 20, + "proximity_boost": { + "weight": 0.5, + "all_pairs": true, + "ordered": false, + "candidates": 10000 + } + } + })"; + expect_equal(source->KnnSearch(query, num_base, sp, nullptr), + immutable->KnnSearch(query, num_base, sp, nullptr)); + expect_equal(source->KnnSearch(query, num_base, sp, nullptr), + immutable_roundtrip->KnnSearch(query, num_base, sp, nullptr)); + expect_equal(source->RangeSearch(query, 1.0F, sp, nullptr), + immutable->RangeSearch(query, 1.0F, sp, nullptr)); + expect_equal(source->RangeSearch(query, 1.0F, sp, nullptr), + immutable_roundtrip->RangeSearch(query, 1.0F, sp, nullptr)); + + // Same retained-prefix bound must hold for the phrase-filter path. Like + // proximity, the immutable backend reaches doc positions via ResolveTerm + + // LookupPositions, whose binary search must also stop at the retained + // prefix. doc 8 lives in the pruned tail, so the phrase must fail + // identically on both backends. bench cannot drive phrase_terms, so this UT + // is the only guard for immutable phrase pruning correctness. + std::string sp_phrase = fmt::format(R"({{ + "sindi": {{ + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.5, + "n_candidate": 20, + "proximity_boost": {{ + "weight": 0.0 + }}, + "phrase_terms": [{}, {}], + "phrase_slop": 5 + }} + }})", + t10, + t20); + expect_equal(source->KnnSearch(query, num_base, sp_phrase, nullptr), + immutable->KnnSearch(query, num_base, sp_phrase, nullptr)); + expect_equal(source->KnnSearch(query, num_base, sp_phrase, nullptr), + immutable_roundtrip->KnnSearch(query, num_base, sp_phrase, nullptr)); + expect_equal(source->RangeSearch(query, 1.0F, sp_phrase, nullptr), + immutable->RangeSearch(query, 1.0F, sp_phrase, nullptr)); + expect_equal(source->RangeSearch(query, 1.0F, sp_phrase, nullptr), + immutable_roundtrip->RangeSearch(query, 1.0F, sp_phrase, nullptr)); +} + +TEST_CASE("SINDI Proximity Benchmark", "[ut][SINDI][Proximity][ProximityBenchmark]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + IndexCommonParam common_param; + common_param.allocator_ = allocator; + + // Generate test data + constexpr uint32_t num_base = 2000; + constexpr uint32_t num_query = 50; + constexpr uint32_t vocab_size = 500; + constexpr uint32_t max_doc_terms = 50; + constexpr uint32_t doc_len = 100; + constexpr int64_t k = 10; + constexpr float beta = 0.3f; + + std::mt19937 rng(42); + + // Generate documents: sparse vectors + token sequences + std::vector> all_ids(num_base); + std::vector> all_vals(num_base); + std::vector> all_seqs(num_base); + std::vector sv_base(num_base); + std::vector base_ids_bench(num_base); + + for (uint32_t i = 0; i < num_base; ++i) { + base_ids_bench[i] = i; + + // Generate token sequence + all_seqs[i].resize(doc_len); + for (uint32_t j = 0; j < doc_len; ++j) { + all_seqs[i][j] = rng() % vocab_size; + } + + // Extract unique terms with random weights + std::set unique; + for (auto t : all_seqs[i]) { + unique.insert(t); + } + uint32_t n_terms = std::min(static_cast(unique.size()), max_doc_terms); + all_ids[i].assign(unique.begin(), unique.end()); + all_ids[i].resize(n_terms); + all_vals[i].resize(n_terms); + for (uint32_t j = 0; j < n_terms; ++j) { + all_vals[i][j] = 0.1f + static_cast(rng() % 90) / 100.0f; + } + + sv_base[i].len_ = n_terms; + sv_base[i].ids_ = all_ids[i].data(); + sv_base[i].vals_ = all_vals[i].data(); + sv_base[i].token_seq_len_ = doc_len; + sv_base[i].token_sequence_ = all_seqs[i].data(); + } + + // Generate queries: pick subset of terms from random docs + std::vector> q_ids_vec(num_query); + std::vector> q_vals_vec(num_query); + std::vector sv_queries(num_query); + + for (uint32_t qi = 0; qi < num_query; ++qi) { + uint32_t src_doc = rng() % num_base; + uint32_t n_q = std::min(static_cast(all_ids[src_doc].size()), 8u); + q_ids_vec[qi].resize(n_q); + q_vals_vec[qi].resize(n_q); + for (uint32_t j = 0; j < n_q; ++j) { + q_ids_vec[qi][j] = all_ids[src_doc][j]; + q_vals_vec[qi][j] = 0.1f + static_cast(rng() % 90) / 100.0f; + } + sv_queries[qi].len_ = n_q; + sv_queries[qi].ids_ = q_ids_vec[qi].data(); + sv_queries[qi].vals_ = q_vals_vec[qi].data(); + } + + // Build index with positions + auto bench_param_str = fmt::format(R"({{ + "use_reorder": false, + "use_quantization": false, + "doc_prune_ratio": 0.0, + "window_size": 10000, + "term_id_limit": {}, + "store_positions": true, + "max_positions_per_term": 64, + "avg_doc_term_length": {} + }})", + vocab_size + 1, + max_doc_terms); + + vsag::JsonType bench_pj = vsag::JsonType::Parse(bench_param_str); + auto bench_ip = std::make_shared(); + bench_ip->FromJson(bench_pj); + + auto bench_index = std::make_unique(bench_ip, common_param); + auto bench_base = vsag::Dataset::Make(); + bench_base->NumElements(num_base) + ->SparseVectors(sv_base.data()) + ->Ids(base_ids_bench.data()) + ->Owner(false); + auto build_res = bench_index->Build(bench_base); + REQUIRE(build_res.size() == 0); + + // Brute-force ground truth with proximity boost + std::vector> gt_ids(num_query); + for (uint32_t qi = 0; qi < num_query; ++qi) { + std::vector> dists; + for (uint32_t di = 0; di < num_base; ++di) { + // Compute IP + float ip = 0.0f; + uint32_t qp = 0, dp = 0; + while (qp < q_ids_vec[qi].size() && dp < all_ids[di].size()) { + if (q_ids_vec[qi][qp] == all_ids[di][dp]) { + ip += q_vals_vec[qi][qp] * all_vals[di][dp]; + qp++; + dp++; + } else if (q_ids_vec[qi][qp] < all_ids[di][dp]) { + qp++; + } else { + dp++; + } + } + if (ip == 0.0f) { + continue; + } + + // Compute proximity boost + std::unordered_map> pos_map; + for (uint32_t p = 0; p < all_seqs[di].size(); ++p) { + pos_map[all_seqs[di][p]].push_back(static_cast(p)); + } + + std::vector> position_lists; + for (uint32_t qj = 0; qj < q_ids_vec[qi].size(); ++qj) { + auto it = pos_map.find(q_ids_vec[qi][qj]); + if (it != pos_map.end()) { + position_lists.push_back(it->second); + } else { + position_lists.emplace_back(); + } + } + + std::vector position_spans; + position_spans.reserve(position_lists.size()); + for (const auto& list : position_lists) { + position_spans.push_back(PosSpan{list.data(), static_cast(list.size())}); + } + float raw_boost = all_pairwise_proximity(position_spans, false); + float pair_count = static_cast(q_ids_vec[qi].size()) * + static_cast(q_ids_vec[qi].size() - 1) / 2.0f; + float norm_boost = (pair_count > 0) ? raw_boost / pair_count : 0.0f; + float boosted_ip = ip * (1.0f + beta * norm_boost); + float dist = 1.0f - boosted_ip; + dists.push_back({dist, static_cast(di)}); + } + std::sort(dists.begin(), dists.end()); + gt_ids[qi].resize(std::min(static_cast(dists.size()), k)); + for (uint64_t j = 0; j < gt_ids[qi].size(); ++j) { + gt_ids[qi][j] = dists[j].second; + } + } + + // Search params + std::string sp_prox = fmt::format(R"({{ + "sindi": {{ + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 50, + "proximity_boost": {{ + "weight": {}, + "boost_multiplicative": true, + "all_pairs": true, + "candidates": 10000 + }} + }} + }})", + beta); + + std::string sp_baseline = R"({ + "sindi": { + "query_prune_ratio": 0.0, + "term_prune_ratio": 0.0, + "n_candidate": 50, + "proximity_boost": { + "weight": 0.0 + } + } + })"; + + // Measure proximity search: recall + QPS + auto query_ds = vsag::Dataset::Make(); + auto t_start_prox = std::chrono::high_resolution_clock::now(); + float total_recall = 0.0f; + + for (uint32_t qi = 0; qi < num_query; ++qi) { + query_ds->NumElements(1)->SparseVectors(sv_queries.data() + qi)->Owner(false); + auto result = bench_index->KnnSearch(query_ds, k, sp_prox, nullptr); + + std::set gt_set(gt_ids[qi].begin(), gt_ids[qi].end()); + uint32_t hits = 0; + for (int64_t j = 0; j < result->GetDim(); ++j) { + if (gt_set.count(result->GetIds()[j]) > 0) { + hits++; + } + } + float recall = gt_set.empty() ? 1.0f : static_cast(hits) / gt_set.size(); + total_recall += recall; + } + auto t_end_prox = std::chrono::high_resolution_clock::now(); + double prox_ms = + std::chrono::duration_cast(t_end_prox - t_start_prox).count() / + 1000.0; + + // Measure baseline search + count rank changes + auto t_start_base = std::chrono::high_resolution_clock::now(); + uint32_t rank_changes = 0; + for (uint32_t qi = 0; qi < num_query; ++qi) { + query_ds->NumElements(1)->SparseVectors(sv_queries.data() + qi)->Owner(false); + auto result_base = bench_index->KnnSearch(query_ds, k, sp_baseline, nullptr); + auto result_prox = bench_index->KnnSearch(query_ds, k, sp_prox, nullptr); + + if (result_base->GetDim() > 0 && result_prox->GetDim() > 0 && + result_base->GetIds()[0] != result_prox->GetIds()[0]) { + rank_changes++; + } + } + auto t_end_base = std::chrono::high_resolution_clock::now(); + double base_ms = + std::chrono::duration_cast(t_end_base - t_start_base).count() / + 1000.0; + + float avg_recall = total_recall / num_query; + double prox_qps = num_query / (prox_ms / 1000.0); + + WARN("=== SINDI Proximity Benchmark ==="); + WARN("Data: " << num_base << " docs, " << num_query << " queries, vocab=" << vocab_size + << ", doc_len=" << doc_len); + WARN("Proximity recall@" << k << " vs brute-force GT: " << avg_recall); + WARN("Proximity search: " << prox_ms << " ms total, QPS=" << static_cast(prox_qps)); + WARN("Baseline+proximity search: " << base_ms << " ms total (" << num_query << " x 2 queries)"); + WARN("Top-1 rank changes (baseline vs proximity): " << rank_changes << "/" << num_query); + WARN("Memory estimate: " << bench_index->EstimateMemory(num_base) << " bytes"); + + REQUIRE(avg_recall > 0.5f); +} diff --git a/src/datacell/sparse_term_datacell.cpp b/src/datacell/sparse_term_datacell.cpp index d8391a7ad3..3b533a019f 100644 --- a/src/datacell/sparse_term_datacell.cpp +++ b/src/datacell/sparse_term_datacell.cpp @@ -17,6 +17,9 @@ #include #include +#include +#include +#include #include "simd/fp16_simd.h" #include "utils/util_functions.h" @@ -262,6 +265,10 @@ SparseTermDataCell::InsertVector(const SparseVector& sparse_base, uint16_t base_ if (term_sizes_[term] == 0) { // create term until needed term_ids_[term] = std::make_unique>(allocator_); term_datas_[term] = std::make_unique>(allocator_); + if (store_positions_) { + term_pos_offsets_[term] = std::make_unique>(allocator_); + term_pos_pool_[term] = std::make_unique>(allocator_); + } } term_ids_[term]->push_back(base_id); @@ -284,6 +291,56 @@ SparseTermDataCell::InsertVector(const SparseVector& sparse_base, uint16_t base_ term_sizes_[term] += 1; } + + // Extract and store positions from token_sequence if enabled + if (store_positions_ && sparse_base.token_sequence_ != nullptr && + sparse_base.token_seq_len_ > 0) { + // Build set of term IDs that survived doc pruning (present in sorted_base) + std::unordered_set surviving_terms; + for (auto& item : sorted_base) { + surviving_terms.insert(item.first); + } + + // Collect positions per surviving term from token_sequence. Positions + // are stored as uint16_t. + std::unordered_map> positions; + constexpr uint32_t kMaxStorablePosition = + static_cast(std::numeric_limits::max()) + 1; + uint32_t limit = std::min(sparse_base.token_seq_len_, kMaxStorablePosition); + for (uint32_t pos = 0; pos < limit; ++pos) { + uint32_t token = sparse_base.token_sequence_[pos]; + if (surviving_terms.count(token) > 0) { + auto& pos_list = positions[token]; + if (pos_list.size() < max_positions_per_term_) { + pos_list.push_back(static_cast(pos)); + } + } + } + + // Append positions to offset+pool for each surviving term + for (auto& item : sorted_base) { + auto term = item.first; + auto& offsets = *term_pos_offsets_[term]; + auto& pool = *term_pos_pool_[term]; + uint32_t pool_start = pool.size(); + offsets.push_back(pool_start); + auto it = positions.find(term); + if (it != positions.end()) { + for (auto p : it->second) { + pool.push_back(p); + } + } + } + } else if (store_positions_) { + // No token_sequence provided: append empty entries for each surviving term + for (auto& item : sorted_base) { + auto term = item.first; + if (term_pos_offsets_[term]) { + term_pos_offsets_[term]->push_back(term_pos_pool_[term]->size()); + } + } + } + total_count_++; } @@ -311,6 +368,15 @@ SparseTermDataCell::ResizeTermList(InnerIdType new_term_capacity) { term_ids_.swap(new_ids); term_datas_.swap(new_datas); term_sizes_.swap(new_sizes); + if (store_positions_) { + Vector>> new_offsets(new_capacity, allocator_); + Vector>> new_pool(new_capacity, allocator_); + std::move(term_pos_offsets_.begin(), term_pos_offsets_.end(), new_offsets.begin()); + std::move(term_pos_pool_.begin(), term_pos_pool_.end(), new_pool.begin()); + term_pos_offsets_.swap(new_offsets); + term_pos_pool_.swap(new_pool); + } + term_capacity_ = new_capacity; } @@ -399,6 +465,8 @@ SparseTermDataCell::GetMemoryUsage() const { auto memory = sizeof(SparseTermDataCell); memory += term_ids_.capacity() * sizeof(std::unique_ptr>); memory += term_datas_.capacity() * sizeof(std::unique_ptr>); + memory += term_pos_offsets_.capacity() * sizeof(std::unique_ptr>); + memory += term_pos_pool_.capacity() * sizeof(std::unique_ptr>); for (const auto& ptr : term_ids_) { if (ptr != nullptr) { memory += sizeof(Vector); @@ -411,6 +479,18 @@ SparseTermDataCell::GetMemoryUsage() const { memory += ptr->capacity() * sizeof(uint8_t); } } + for (const auto& ptr : term_pos_offsets_) { + if (ptr != nullptr) { + memory += sizeof(Vector); + memory += ptr->capacity() * sizeof(uint32_t); + } + } + for (const auto& ptr : term_pos_pool_) { + if (ptr != nullptr) { + memory += sizeof(Vector); + memory += ptr->capacity() * sizeof(uint16_t); + } + } memory += sizeof(QuantizationParams); memory += term_sizes_.capacity() * sizeof(uint32_t); return static_cast(memory); @@ -496,6 +576,22 @@ SparseTermDataCell::Serialize(StreamWriter& writer) const { } } StreamWriter::WriteVector(writer, term_sizes_); + + // Serialize position data if enabled + if (store_positions_) { + StreamWriter::WriteObj(writer, static_cast(1)); // position data marker + Vector empty_offsets(allocator_); + Vector empty_pool(allocator_); + for (uint32_t i = 0; i < term_capacity_; i++) { + if (term_pos_offsets_[i] && !term_pos_offsets_[i]->empty()) { + StreamWriter::WriteVector(writer, *term_pos_offsets_[i]); + StreamWriter::WriteVector(writer, *term_pos_pool_[i]); + } else { + StreamWriter::WriteVector(writer, empty_offsets); + StreamWriter::WriteVector(writer, empty_pool); + } + } + } } void @@ -541,6 +637,61 @@ SparseTermDataCell::Deserialize(StreamReader& reader) { } } } + + // Deserialize position data if enabled + if (store_positions_) { + uint8_t marker; + StreamReader::ReadObj(reader, marker); + if (marker == 1) { + Vector offsets_buf(allocator_); + Vector pool_buf(allocator_); + for (uint32_t i = 0; i < term_capacity; i++) { + StreamReader::ReadVector(reader, offsets_buf); + StreamReader::ReadVector(reader, pool_buf); + if (!offsets_buf.empty()) { + term_pos_offsets_[i] = std::make_unique>(allocator_); + term_pos_pool_[i] = std::make_unique>(allocator_); + *term_pos_offsets_[i] = std::move(offsets_buf); + *term_pos_pool_[i] = std::move(pool_buf); + offsets_buf = Vector(allocator_); + pool_buf = Vector(allocator_); + } else { + offsets_buf.clear(); + pool_buf.clear(); + } + } + } + } +} + +std::vector +SparseTermDataCell::GetPositions(uint32_t term_id, uint32_t posting_index) const { + if (!store_positions_ || term_id >= term_capacity_ || !term_pos_offsets_[term_id] || + posting_index >= term_pos_offsets_[term_id]->size()) { + return {}; + } + + auto& offsets = *term_pos_offsets_[term_id]; + auto& pool = *term_pos_pool_[term_id]; + uint32_t start = offsets[posting_index]; + uint32_t end = (posting_index + 1 < offsets.size()) ? offsets[posting_index + 1] : pool.size(); + + return std::vector(pool.begin() + start, pool.begin() + end); +} + +std::pair +SparseTermDataCell::GetPositionsView(uint32_t term_id, uint32_t posting_index) const { + if (!store_positions_ || term_id >= term_capacity_ || !term_pos_offsets_[term_id] || + posting_index >= term_pos_offsets_[term_id]->size()) { + return {nullptr, 0}; + } + + auto& offsets = *term_pos_offsets_[term_id]; + auto& pool = *term_pos_pool_[term_id]; + uint32_t start = offsets[posting_index]; + uint32_t end = (posting_index + 1 < offsets.size()) ? offsets[posting_index + 1] : pool.size(); + + return {pool.data() + start, end - start}; } void diff --git a/src/datacell/sparse_term_datacell.h b/src/datacell/sparse_term_datacell.h index bc98922f28..3f77aac4ee 100644 --- a/src/datacell/sparse_term_datacell.h +++ b/src/datacell/sparse_term_datacell.h @@ -35,7 +35,9 @@ class SparseTermDataCell { uint32_t term_id_limit, Allocator* allocator, SparseValueQuantizationType sparse_value_quant_type, - std::shared_ptr quantization_params) + std::shared_ptr quantization_params, + bool store_positions = false, + uint32_t max_positions_per_term = 64) : doc_retain_ratio_(doc_retain_ratio), term_id_limit_(term_id_limit), allocator_(allocator), @@ -43,9 +45,19 @@ class SparseTermDataCell { term_datas_(allocator), term_sizes_(allocator), sparse_value_quant_type_(sparse_value_quant_type), - quantization_params_(std::move(quantization_params)) { + quantization_params_(std::move(quantization_params)), + store_positions_(store_positions), + max_positions_per_term_(max_positions_per_term), + term_pos_offsets_(allocator), + term_pos_pool_(allocator) { } + /** + * @brief Accumulate IP scores for the query. + * + * @param global_dists Distance array to accumulate IP scores + * @param computer SparseTermComputer for query processing + */ void Query(float* global_dists, const SparseTermComputerPtr& computer) const; @@ -91,6 +103,18 @@ class SparseTermDataCell { void InsertVector(const SparseVector& sparse_base, uint16_t base_id); + // Get positions for a given term and doc posting index. + // Returns empty vector if positions are not stored or term/doc not found. + std::vector + GetPositions(uint32_t term_id, uint32_t posting_index) const; + + // Zero-copy view into the position pool for a given term and doc posting + // index. Returns {data, size} pointing into term_pos_pool_; the data is + // valid for read only as long as the pool is not mutated (i.e. during a + // query). Returns {nullptr, 0} if positions are not stored or not found. + std::pair + GetPositionsView(uint32_t term_id, uint32_t posting_index) const; + void ResizeTermList(InnerIdType new_term_capacity); @@ -159,5 +183,13 @@ class SparseTermDataCell { int64_t total_count_{0}; std::shared_ptr quantization_params_; + + // Position storage (opt-in via store_positions_) + bool store_positions_{false}; + uint32_t max_positions_per_term_{64}; + // Per-term offset arrays: term_pos_offsets_[term][i] = start index in pool for posting i + Vector>> term_pos_offsets_; + // Per-term position pools: term_pos_pool_[term] = flat array of all positions for this term + Vector>> term_pos_pool_; }; } // namespace vsag diff --git a/src/datacell/sparse_term_datacell_position_test.cpp b/src/datacell/sparse_term_datacell_position_test.cpp new file mode 100644 index 0000000000..3017656f75 --- /dev/null +++ b/src/datacell/sparse_term_datacell_position_test.cpp @@ -0,0 +1,365 @@ + +// Copyright 2024-present the vsag project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "impl/allocator/safe_allocator.h" +#include "sparse_term_datacell.h" +#include "storage/stream_reader.h" +#include "storage/stream_writer.h" +#include "unittest.h" + +using namespace vsag; + +TEST_CASE("SparseTermDataCell Position Storage Basic", "[ut][SparseTermDatacell][Position]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + float doc_retain_ratio = 1.0f; // no pruning + uint32_t term_id_limit = 100; + bool store_positions = true; + uint32_t max_positions_per_term = 64; + + SparseTermDataCell cell(doc_retain_ratio, + term_id_limit, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + store_positions, + max_positions_per_term); + + // Document: "A(10) B(20) C(30) A(10) D(40)" + // Sparse vector: ids=[10,20,30,40], vals=[1,1,1,1] + // token_seq: [10, 20, 30, 10, 40] + uint32_t ids[] = {10, 20, 30, 40}; + float vals[] = {1.0f, 1.0f, 1.0f, 1.0f}; + uint32_t token_seq[] = {10, 20, 30, 10, 40}; + + SparseVector sv; + sv.len_ = 4; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = 5; + sv.token_sequence_ = token_seq; + + cell.InsertVector(sv, 0); + + // Verify positions stored correctly + // term 10: positions [0, 3] + auto pos_10 = cell.GetPositions(10, 0); + REQUIRE(pos_10.size() == 2); + REQUIRE(pos_10[0] == 0); + REQUIRE(pos_10[1] == 3); + + // term 20: positions [1] + auto pos_20 = cell.GetPositions(20, 0); + REQUIRE(pos_20.size() == 1); + REQUIRE(pos_20[0] == 1); + + // term 30: positions [2] + auto pos_30 = cell.GetPositions(30, 0); + REQUIRE(pos_30.size() == 1); + REQUIRE(pos_30[0] == 2); + + // term 40: positions [4] + auto pos_40 = cell.GetPositions(40, 0); + REQUIRE(pos_40.size() == 1); + REQUIRE(pos_40[0] == 4); +} + +TEST_CASE("SparseTermDataCell Position Storage Cap", "[ut][SparseTermDatacell][Position]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + bool store_positions = true; + uint32_t max_positions_per_term = 3; // very small cap + + SparseTermDataCell cell(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + store_positions, + max_positions_per_term); + + // term 10 appears 5 times but cap is 3 + uint32_t ids[] = {10}; + float vals[] = {1.0f}; + uint32_t token_seq[] = {10, 10, 10, 10, 10}; + + SparseVector sv; + sv.len_ = 1; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = 5; + sv.token_sequence_ = token_seq; + + cell.InsertVector(sv, 0); + + auto pos = cell.GetPositions(10, 0); + REQUIRE(pos.size() == 3); // capped + REQUIRE(pos[0] == 0); + REQUIRE(pos[1] == 1); + REQUIRE(pos[2] == 2); +} + +TEST_CASE("SparseTermDataCell Position Storage Caps At Uint16 Range", + "[ut][SparseTermDatacell][Position]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + bool store_positions = true; + + SparseTermDataCell cell(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + store_positions, + 70000); + + uint32_t ids[] = {10}; + float vals[] = {1.0f}; + std::vector token_seq(65537, 10); + + SparseVector sv; + sv.len_ = 1; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = static_cast(token_seq.size()); + sv.token_sequence_ = token_seq.data(); + + cell.InsertVector(sv, 0); + + auto pos = cell.GetPositions(10, 0); + REQUIRE(pos.size() == 65536); + REQUIRE(pos.front() == 0); + REQUIRE(pos.back() == std::numeric_limits::max()); +} + +TEST_CASE("SparseTermDataCell Position Storage Disabled", "[ut][SparseTermDatacell][Position]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + bool store_positions = false; + + SparseTermDataCell cell(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + store_positions, + 64); + + uint32_t ids[] = {10, 20}; + float vals[] = {1.0f, 1.0f}; + uint32_t token_seq[] = {10, 20, 10}; + + SparseVector sv; + sv.len_ = 2; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = 3; + sv.token_sequence_ = token_seq; + + cell.InsertVector(sv, 0); + + // No positions stored + auto pos = cell.GetPositions(10, 0); + REQUIRE(pos.empty()); +} + +TEST_CASE("SparseTermDataCell Position Storage Memory Usage", + "[ut][SparseTermDatacell][Position]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + SparseTermDataCell without_positions(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + false, + 64); + SparseTermDataCell with_positions(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + true, + 64); + + uint32_t ids[] = {10, 20}; + float vals[] = {1.0f, 1.0f}; + uint32_t token_seq[] = {10, 20, 10}; + SparseVector vector; + vector.len_ = 2; + vector.ids_ = ids; + vector.vals_ = vals; + vector.token_seq_len_ = 3; + vector.token_sequence_ = token_seq; + + without_positions.InsertVector(vector, 0); + with_positions.InsertVector(vector, 0); + + REQUIRE(with_positions.GetMemoryUsage() > without_positions.GetMemoryUsage()); +} + +TEST_CASE("SparseTermDataCell Position Storage No TokenSeq", "[ut][SparseTermDatacell][Position]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + bool store_positions = true; + + SparseTermDataCell cell(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + store_positions, + 64); + + uint32_t ids[] = {10, 20}; + float vals[] = {1.0f, 1.0f}; + + SparseVector sv; + sv.len_ = 2; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = 0; + sv.token_sequence_ = nullptr; + + cell.InsertVector(sv, 0); + + // No positions stored because token_sequence is null + auto pos = cell.GetPositions(10, 0); + REQUIRE(pos.empty()); +} + +TEST_CASE("SparseTermDataCell Position Storage Multiple Docs", + "[ut][SparseTermDatacell][Position]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + bool store_positions = true; + + SparseTermDataCell cell(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + store_positions, + 64); + + // Doc 0: term 10 at pos [0, 2] + { + uint32_t ids[] = {10, 20}; + float vals[] = {1.0f, 1.0f}; + uint32_t token_seq[] = {10, 20, 10}; + SparseVector sv; + sv.len_ = 2; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = 3; + sv.token_sequence_ = token_seq; + cell.InsertVector(sv, 0); + } + + // Doc 1: term 10 at pos [1] + { + uint32_t ids[] = {10, 30}; + float vals[] = {1.0f, 1.0f}; + uint32_t token_seq[] = {30, 10}; + SparseVector sv; + sv.len_ = 2; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = 2; + sv.token_sequence_ = token_seq; + cell.InsertVector(sv, 1); + } + + // Doc 0, term 10: [0, 2] + auto pos0 = cell.GetPositions(10, 0); + REQUIRE(pos0.size() == 2); + REQUIRE(pos0[0] == 0); + REQUIRE(pos0[1] == 2); + + // Doc 1, term 10: [1] + auto pos1 = cell.GetPositions(10, 1); + REQUIRE(pos1.size() == 1); + REQUIRE(pos1[0] == 1); +} + +TEST_CASE("SparseTermDataCell Position Serialize Deserialize", + "[ut][SparseTermDatacell][Position]") { + auto allocator = SafeAllocator::FactoryDefaultAllocator(); + bool store_positions = true; + + SparseTermDataCell cell(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + store_positions, + 64); + + // Insert two docs + { + uint32_t ids[] = {10, 20}; + float vals[] = {1.0f, 2.0f}; + uint32_t token_seq[] = {10, 20, 10}; + SparseVector sv; + sv.len_ = 2; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = 3; + sv.token_sequence_ = token_seq; + cell.InsertVector(sv, 0); + } + { + uint32_t ids[] = {10}; + float vals[] = {3.0f}; + uint32_t token_seq[] = {10}; + SparseVector sv; + sv.len_ = 1; + sv.ids_ = ids; + sv.vals_ = vals; + sv.token_seq_len_ = 1; + sv.token_sequence_ = token_seq; + cell.InsertVector(sv, 1); + } + + // Serialize + std::vector buffer(1024 * 1024); + BufferStreamWriter writer(buffer.data()); + cell.Serialize(writer); + + // Deserialize into a new cell + SparseTermDataCell cell2(1.0f, + 100, + allocator.get(), + SparseValueQuantizationType::FP32, + nullptr, + store_positions, + 64); + auto reader_func = [&buffer](uint64_t offset, uint64_t size, void* dest) { + memcpy(dest, buffer.data() + offset, size); + }; + ReadFuncStreamReader reader(reader_func, 0, writer.GetCursor()); + cell2.Deserialize(reader); + + // Verify positions survive round-trip + auto pos0 = cell2.GetPositions(10, 0); + REQUIRE(pos0.size() == 2); + REQUIRE(pos0[0] == 0); + REQUIRE(pos0[1] == 2); + + auto pos1 = cell2.GetPositions(10, 1); + REQUIRE(pos1.size() == 1); + REQUIRE(pos1[0] == 0); + + auto pos_20 = cell2.GetPositions(20, 0); + REQUIRE(pos_20.size() == 1); + REQUIRE(pos_20[0] == 1); +} diff --git a/src/inner_string_params.h b/src/inner_string_params.h index 2b8ac532ba..1b1d9e08e8 100644 --- a/src/inner_string_params.h +++ b/src/inner_string_params.h @@ -125,6 +125,16 @@ const char* const SPARSE_DESERIALIZE_WITHOUT_BUFFER = "deserialize_without_buffe const char* const SPARSE_AVG_DOC_TERM_LENGTH = "avg_doc_term_length"; const char* const SPARSE_REMAP_TERM_IDS = "remap_term_ids"; const char* const SPARSE_IMMUTABLE = "immutable"; +const char* const SPARSE_STORE_POSITIONS = "store_positions"; +const char* const SPARSE_MAX_POSITIONS_PER_TERM = "max_positions_per_term"; +const char* const SPARSE_PROXIMITY_BOOST = "proximity_boost"; +const char* const SPARSE_PROXIMITY_BOOST_CANDIDATES = "candidates"; +const char* const SPARSE_PROXIMITY_BOOST_WEIGHT = "weight"; +const char* const SPARSE_PROXIMITY_BOOST_ORDERED = "ordered"; +const char* const SPARSE_PROXIMITY_BOOST_MULTIPLICATIVE = "boost_multiplicative"; +const char* const SPARSE_PROXIMITY_BOOST_ALL_PAIRS = "all_pairs"; +const char* const SPARSE_PHRASE_TERMS = "phrase_terms"; +const char* const SPARSE_PHRASE_SLOP = "phrase_slop"; // graph param value const char* const GRAPH_PARAM_MAX_DEGREE_KEY = "max_degree"; diff --git a/tools/eval/eval_dataset.h b/tools/eval/eval_dataset.h index 8aa85341fb..1d5b14a755 100644 --- a/tools/eval/eval_dataset.h +++ b/tools/eval/eval_dataset.h @@ -245,6 +245,11 @@ class EvalDataset { } } + bool + HasTokenSequences() const { + return has_token_sequences_; + } + private: using shape_t = std::pair; static std::unordered_set @@ -325,6 +330,8 @@ class EvalDataset { std::vector train_token_seq_offsets_; std::vector test_token_seq_offsets_; + bool has_token_sequences_{false}; + std::vector multi_train_vectors_; std::vector multi_test_vectors_; std::shared_ptr train_vector_counts_;