From e18c14b1e0c4e7f2d7d2aec9b4ae3dd5fd7be852 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Mon, 8 Jun 2026 17:35:09 +0800 Subject: [PATCH 01/12] update params --- src/binding/params.cc | 445 ++++++++++++++++++++++++++++++++---------- src/binding/params.h | 43 ++-- src/zvec | 2 +- 3 files changed, 373 insertions(+), 117 deletions(-) diff --git a/src/binding/params.cc b/src/binding/params.cc index 1d1f640..8fd01a8 100644 --- a/src/binding/params.cc +++ b/src/binding/params.cc @@ -29,12 +29,16 @@ zvec::Result ParseIndexParams( return ParseFlatIndexParams(obj); case zvec::IndexType::HNSW: return ParseHnswIndexParams(obj); - case zvec::IndexType::IVF: - return ParseIVFIndexParams(obj); case zvec::IndexType::HNSW_RABITQ: return ParseHnswRabitqIndexParams(obj); + case zvec::IndexType::IVF: + return ParseIVFIndexParams(obj); + case zvec::IndexType::DISKANN: + return ParseDiskAnnIndexParams(obj); case zvec::IndexType::INVERT: return ParseInvertIndexParams(obj); + case zvec::IndexType::FTS: + return ParseFtsIndexParams(obj); default: uint32_t type_val = static_cast(index_type); return tl::make_unexpected(zvec::Status::InvalidArgument( @@ -49,12 +53,16 @@ Napi::Object CreateIndexParams(Napi::Env env, zvec::IndexParams::Ptr params) { return CreateFlatIndexParams(env, params); case zvec::IndexType::HNSW: return CreateHnswIndexParams(env, params); - case zvec::IndexType::IVF: - return CreateIVFIndexParams(env, params); case zvec::IndexType::HNSW_RABITQ: return CreateHnswRabitqIndexParams(env, params); + case zvec::IndexType::IVF: + return CreateIVFIndexParams(env, params); + case zvec::IndexType::DISKANN: + return CreateDiskAnnIndexParams(env, params); case zvec::IndexType::INVERT: return CreateInvertIndexParams(env, params); + case zvec::IndexType::FTS: + return CreateFtsIndexParams(env, params); default: return Napi::Object::New(env); } @@ -140,8 +148,19 @@ zvec::Result ParseHnswIndexParams( } } + bool use_contiguous_memory{false}; + if (obj.Has("useContiguousMemory")) { + if (obj.Get("useContiguousMemory").IsBoolean()) { + use_contiguous_memory = + obj.Get("useContiguousMemory").As().Value(); + } else { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a boolean for 'useContiguousMemory'")); + } + } + return std::make_shared( - metric_type, m, ef_construction, quantize_type); + metric_type, m, ef_construction, quantize_type, use_contiguous_memory); } @@ -154,6 +173,90 @@ Napi::Object CreateHnswIndexParams(Napi::Env env, obj.Set("m", hnsw_params->m()); obj.Set("efConstruction", hnsw_params->ef_construction()); obj.Set("quantizeType", static_cast(hnsw_params->quantize_type())); + obj.Set("useContiguousMemory", hnsw_params->use_contiguous_memory()); + return obj; +} + + +zvec::Result ParseHnswRabitqIndexParams( + const Napi::Object &obj) { + zvec::MetricType metric_type{zvec::MetricType::IP}; + if (obj.Has("metricType")) { + auto parsed_metric_type = ParseMetricType(obj.Get("metricType")); + if (parsed_metric_type) { + metric_type = parsed_metric_type.value(); + } else { + return tl::make_unexpected(parsed_metric_type.error()); + } + } + + int total_bits{7}; + if (obj.Has("totalBits")) { + if (obj.Get("totalBits").IsNumber()) { + total_bits = obj.Get("totalBits").As(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'totalBits'")); + } + } + + int num_clusters{16}; + if (obj.Has("numClusters")) { + if (obj.Get("numClusters").IsNumber()) { + num_clusters = obj.Get("numClusters").As(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'numClusters'")); + } + } + + int m{50}; + if (obj.Has("m")) { + if (obj.Get("m").IsNumber()) { + m = obj.Get("m").As(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'm'")); + } + } + + int ef_construction{500}; + if (obj.Has("efConstruction")) { + if (obj.Get("efConstruction").IsNumber()) { + ef_construction = obj.Get("efConstruction").As(); + } else { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a number for 'efConstruction'")); + } + } + + int sample_count{0}; + if (obj.Has("sampleCount")) { + if (obj.Get("sampleCount").IsNumber()) { + sample_count = obj.Get("sampleCount").As(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'sampleCount'")); + } + } + + return std::make_shared( + metric_type, total_bits, num_clusters, m, ef_construction, sample_count); +} + + +Napi::Object CreateHnswRabitqIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params) { + auto obj = Napi::Object::New(env); + auto rabitq_params = + std::dynamic_pointer_cast(params); + obj.Set("indexType", static_cast(rabitq_params->type())); + obj.Set("metricType", static_cast(rabitq_params->metric_type())); + obj.Set("totalBits", rabitq_params->total_bits()); + obj.Set("numClusters", rabitq_params->num_clusters()); + obj.Set("m", rabitq_params->m()); + obj.Set("efConstruction", rabitq_params->ef_construction()); + obj.Set("sampleCount", rabitq_params->sample_count()); return obj; } @@ -220,7 +323,7 @@ Napi::Object CreateIVFIndexParams(Napi::Env env, } -zvec::Result ParseHnswRabitqIndexParams( +zvec::Result ParseDiskAnnIndexParams( const Napi::Object &obj) { zvec::MetricType metric_type{zvec::MetricType::IP}; if (obj.Has("metricType")) { @@ -232,73 +335,63 @@ zvec::Result ParseHnswRabitqIndexParams( } } - int total_bits{7}; - if (obj.Has("totalBits")) { - if (obj.Get("totalBits").IsNumber()) { - total_bits = obj.Get("totalBits").As(); + int max_degree{100}; + if (obj.Has("maxDegree")) { + if (obj.Get("maxDegree").IsNumber()) { + max_degree = obj.Get("maxDegree").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'totalBits'")); + zvec::Status::InvalidArgument("Expected a number for 'maxDegree'")); } } - int num_clusters{16}; - if (obj.Has("numClusters")) { - if (obj.Get("numClusters").IsNumber()) { - num_clusters = obj.Get("numClusters").As(); + int list_size{50}; + if (obj.Has("listSize")) { + if (obj.Get("listSize").IsNumber()) { + list_size = obj.Get("listSize").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'numClusters'")); + zvec::Status::InvalidArgument("Expected a number for 'listSize'")); } } - int m{50}; - if (obj.Has("m")) { - if (obj.Get("m").IsNumber()) { - m = obj.Get("m").As(); + int pq_chunk_num{0}; + if (obj.Has("pqChunkNum")) { + if (obj.Get("pqChunkNum").IsNumber()) { + pq_chunk_num = obj.Get("pqChunkNum").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'm'")); + zvec::Status::InvalidArgument("Expected a number for 'pqChunkNum'")); } } - int ef_construction{500}; - if (obj.Has("efConstruction")) { - if (obj.Get("efConstruction").IsNumber()) { - ef_construction = obj.Get("efConstruction").As(); - } else { - return tl::make_unexpected(zvec::Status::InvalidArgument( - "Expected a number for 'efConstruction'")); - } - } - - int sample_count{0}; - if (obj.Has("sampleCount")) { - if (obj.Get("sampleCount").IsNumber()) { - sample_count = obj.Get("sampleCount").As(); + zvec::QuantizeType quantize_type{zvec::QuantizeType::UNDEFINED}; + if (obj.Has("quantizeType")) { + auto parsed_quantize_type = ParseQuantizeType(obj.Get("quantizeType")); + if (parsed_quantize_type) { + quantize_type = parsed_quantize_type.value(); } else { - return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'sampleCount'")); + return tl::make_unexpected(parsed_quantize_type.error()); } } - return std::make_shared( - metric_type, total_bits, num_clusters, m, ef_construction, sample_count); + return std::make_shared( + metric_type, max_degree, list_size, pq_chunk_num, quantize_type); } -Napi::Object CreateHnswRabitqIndexParams(Napi::Env env, - zvec::IndexParams::Ptr params) { +Napi::Object CreateDiskAnnIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params) { auto obj = Napi::Object::New(env); - auto rabitq_params = - std::dynamic_pointer_cast(params); - obj.Set("indexType", static_cast(rabitq_params->type())); - obj.Set("metricType", static_cast(rabitq_params->metric_type())); - obj.Set("totalBits", rabitq_params->total_bits()); - obj.Set("numClusters", rabitq_params->num_clusters()); - obj.Set("m", rabitq_params->m()); - obj.Set("efConstruction", rabitq_params->ef_construction()); - obj.Set("sampleCount", rabitq_params->sample_count()); + auto diskann_params = + std::dynamic_pointer_cast(params); + obj.Set("indexType", static_cast(diskann_params->type())); + obj.Set("metricType", static_cast(diskann_params->metric_type())); + obj.Set("maxDegree", diskann_params->max_degree()); + obj.Set("listSize", diskann_params->list_size()); + obj.Set("pqChunkNum", diskann_params->pq_chunk_num()); + obj.Set("quantizeType", + static_cast(diskann_params->quantize_type())); return obj; } @@ -343,24 +436,87 @@ Napi::Object CreateInvertIndexParams(Napi::Env env, } -zvec::Result ParseVectorQuery( +zvec::Result> ParseFtsIndexParams( + const Napi::Object &obj) { + std::string tokenizer_name{"standard"}; + if (obj.Has("tokenizerName")) { + if (obj.Get("tokenizerName").IsString()) { + tokenizer_name = obj.Get("tokenizerName").As().Utf8Value(); + } else { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a string for 'tokenizerName'")); + } + } + + std::vector filters{"lowercase"}; + if (obj.Has("filters")) { + if (!obj.Get("filters").IsArray()) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected an array of strings for 'filters'")); + } + filters.clear(); + Napi::Array array = obj.Get("filters").As(); + for (uint32_t i = 0; i < array.Length(); i++) { + auto item = array.Get(i); + if (!item.IsString()) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected an array of strings for 'filters'")); + } + filters.emplace_back(item.As().Utf8Value()); + } + } + + std::string extra_params{}; + if (obj.Has("extraParams")) { + if (obj.Get("extraParams").IsString()) { + extra_params = obj.Get("extraParams").As().Utf8Value(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a string for 'extraParams'")); + } + } + + return std::make_shared(tokenizer_name, filters, + extra_params); +} + + +Napi::Object CreateFtsIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params) { + auto obj = Napi::Object::New(env); + auto fts_params = std::dynamic_pointer_cast(params); + obj.Set("indexType", static_cast(fts_params->type())); + obj.Set("tokenizerName", fts_params->tokenizer_name()); + auto filters = Napi::Array::New(env, fts_params->filters().size()); + for (size_t i = 0; i < fts_params->filters().size(); i++) { + filters.Set(i, fts_params->filters()[i]); + } + obj.Set("filters", filters); + obj.Set("extraParams", fts_params->extra_params()); + return obj; +} + + +zvec::Result ParseSearchQuery( const Napi::Value &value, zvec::CollectionSchema::Ptr schema) { if (!value.IsObject()) { return tl::make_unexpected( zvec::Status::InvalidArgument("Expected an object for Query")); } - zvec::VectorQuery query{}; + zvec::SearchQuery query{}; auto obj = value.As(); zvec::FieldSchema *field_schema{nullptr}; if (obj.Has("fieldName")) { if (obj.Get("fieldName").IsString()) { - query.field_name_ = obj.Get("fieldName").As().Utf8Value(); - field_schema = schema->get_field(query.field_name_); + query.target_.field_name_ = + obj.Get("fieldName").As().Utf8Value(); + field_schema = schema->get_field(query.target_.field_name_); if (!field_schema) { - return tl::make_unexpected(zvec::Status::InvalidArgument( - "'", query.field_name_, "' not found in collection schema")); + return tl::make_unexpected( + zvec::Status::InvalidArgument("'", query.target_.field_name_, + "' not found in collection schema")); } } else { return tl::make_unexpected(zvec::Status::InvalidArgument( @@ -376,7 +532,19 @@ zvec::Result ParseVectorQuery( zvec::Status::InvalidArgument("Expected a number for 'topk'")); } } - if (obj.Has("vector")) { + + const bool has_vector = obj.Has("vector") && + !obj.Get("vector").IsUndefined() && + !obj.Get("vector").IsNull(); + const bool has_fts = obj.Has("fts") && !obj.Get("fts").IsUndefined() && + !obj.Get("fts").IsNull(); + + if (has_vector && has_fts) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Cannot combine 'vector' and 'fts' in a single Query")); + } + + if (has_vector) { if (!field_schema) { return tl::make_unexpected(zvec::Status::InvalidArgument( "A vector name must be provided when performing vector queries")); @@ -392,27 +560,32 @@ zvec::Result ParseVectorQuery( } } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Field '", query.field_name_, + zvec::Status::InvalidArgument("Field '", query.target_.field_name_, "' is not a vector field and cannot be " "queried using vector operations.")); } - if (obj.Has("params")) { - auto parsed_params = ParseQueryParams(obj.Get("params")); - if (parsed_params) { - if (parsed_params.value()->type() == field_schema->index_type()) { - query.query_params_ = parsed_params.value(); - } else { - return tl::make_unexpected(zvec::Status::InvalidArgument( - "Unmatched queryParams type in Query")); - } - } else { - return tl::make_unexpected(parsed_params.error()); - } - } - } else { - if (field_schema) { + } else if (has_fts) { + if (!field_schema) { return tl::make_unexpected(zvec::Status::InvalidArgument( - "A vector must be provided when performing vector queries")); + "A field name must be provided when performing FTS queries")); + } + auto parsed_fts = ParseFtsClause(obj.Get("fts")); + if (parsed_fts) { + query.target_.clause_ = std::move(parsed_fts.value()); + } else { + return tl::make_unexpected(parsed_fts.error()); + } + } else if (field_schema) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "A vector or FTS clause must be provided when 'fieldName' is set")); + } + + if (obj.Has("params")) { + auto parsed_params = ParseQueryParams(obj.Get("params")); + if (parsed_params) { + query.target_.query_params_ = parsed_params.value(); + } else { + return tl::make_unexpected(parsed_params.error()); } } if (obj.Has("filter")) { @@ -456,7 +629,7 @@ zvec::Result ParseVectorQuery( zvec::Status ParseVectorToString(const Napi::Value &value, zvec::FieldSchema *schema, - zvec::VectorQuery *query) { + zvec::SearchQuery *query) { if (value.IsTypedArray()) { auto array = value.As(); auto ta_type = array.TypedArrayType(); @@ -476,7 +649,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, std::memcpy(buf.data() + (i * sizeof(uint16_t)), &val, sizeof(uint16_t)); } - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } case zvec::DataType::VECTOR_FP32: { @@ -489,7 +662,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, std::string buf; buf.resize(ta_length * sizeof(float)); std::memcpy(buf.data(), float32Array.Data(), ta_length * sizeof(float)); - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } case zvec::DataType::VECTOR_INT8: { @@ -501,7 +674,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, std::string buf; buf.resize(ta_length * sizeof(int8_t)); std::memcpy(buf.data(), int8Array.Data(), ta_length * sizeof(int8_t)); - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } default: { @@ -528,7 +701,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, "' must be numbers"); } } - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } case zvec::DataType::VECTOR_FP32: { @@ -545,7 +718,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, "' must be numbers"); } } - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } case zvec::DataType::VECTOR_INT8: { @@ -563,7 +736,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, "' must be numbers"); } } - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } default: { @@ -582,7 +755,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, zvec::Status ParseVectorToMap(const Napi::Value &value, zvec::FieldSchema *schema, - zvec::VectorQuery *query) { + zvec::SearchQuery *query) { if (!value.IsObject()) { return zvec::Status::InvalidArgument("Expected sparse vector[", schema->name(), "] to be an object"); @@ -623,8 +796,7 @@ zvec::Status ParseVectorToMap(const Napi::Value &value, indices_ptr[i] = index; values_ptr[i] = vectorValue.As().FloatValue(); } - query->query_sparse_indices_ = std::move(indices); - query->query_sparse_values_ = std::move(values); + query->target_.set_sparse_vector(std::move(indices), std::move(values)); break; } case zvec::DataType::SPARSE_VECTOR_FP16: { @@ -658,8 +830,7 @@ zvec::Status ParseVectorToMap(const Napi::Value &value, zvec::ailego::Float16 val = vectorValue.As().FloatValue(); values_ptr[i] = val; } - query->query_sparse_indices_ = std::move(indices); - query->query_sparse_values_ = std::move(values); + query->target_.set_sparse_vector(std::move(indices), std::move(values)); break; } default: { @@ -671,6 +842,33 @@ zvec::Status ParseVectorToMap(const Napi::Value &value, } +zvec::Result ParseFtsClause(const Napi::Value &value) { + if (!value.IsObject()) { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected an object for 'fts'")); + } + auto obj = value.As(); + zvec::FtsClause fts_clause{}; + if (obj.Has("queryString")) { + if (!obj.Get("queryString").IsString()) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a string for 'queryString' in 'fts'")); + } + fts_clause.query_string_ = + obj.Get("queryString").As().Utf8Value(); + } + if (obj.Has("matchString")) { + if (!obj.Get("matchString").IsString()) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a string for 'matchString' in 'fts'")); + } + fts_clause.match_string_ = + obj.Get("matchString").As().Utf8Value(); + } + return fts_clause; +} + + zvec::Result ParseQueryParams( const Napi::Value &value) { if (!value.IsObject()) { @@ -696,13 +894,17 @@ zvec::Result ParseQueryParams( "Flat index type is not allowed for QueryParams")); case zvec::IndexType::HNSW: return ParseHnswQueryParams(obj); - case zvec::IndexType::IVF: - return ParseIVFQueryParams(obj); case zvec::IndexType::HNSW_RABITQ: return ParseHnswRabitqQueryParams(obj); + case zvec::IndexType::IVF: + return ParseIVFQueryParams(obj); + case zvec::IndexType::DISKANN: + return ParseDiskAnnQueryParams(obj); case zvec::IndexType::INVERT: return tl::make_unexpected(zvec::Status::InvalidArgument( "Inverted index type is not allowed for QueryParams")); + case zvec::IndexType::FTS: + return ParseFtsQueryParams(obj); default: uint32_t type_val = static_cast(query_type); return tl::make_unexpected(zvec::Status::InvalidArgument( @@ -764,15 +966,15 @@ zvec::Result ParseHnswQueryParams( } -zvec::Result ParseIVFQueryParams( +zvec::Result ParseHnswRabitqQueryParams( const Napi::Object &obj) { - int nprobe{10}; - if (obj.Has("nprobe")) { - if (obj.Get("nprobe").IsNumber()) { - nprobe = obj.Get("nprobe").As().Int32Value(); + int ef{300}; + if (obj.Has("ef")) { + if (obj.Get("ef").IsNumber()) { + ef = obj.Get("ef").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'nprobe'")); + zvec::Status::InvalidArgument("Expected a number for 'ef'")); } } @@ -806,19 +1008,20 @@ zvec::Result ParseIVFQueryParams( } } - return std::make_shared(nprobe, is_using_refiner); + return std::make_shared(ef, radius, is_linear, + is_using_refiner); } -zvec::Result ParseHnswRabitqQueryParams( +zvec::Result ParseIVFQueryParams( const Napi::Object &obj) { - int ef{300}; - if (obj.Has("ef")) { - if (obj.Get("ef").IsNumber()) { - ef = obj.Get("ef").As().Int32Value(); + int nprobe{10}; + if (obj.Has("nprobe")) { + if (obj.Get("nprobe").IsNumber()) { + nprobe = obj.Get("nprobe").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'ef'")); + zvec::Status::InvalidArgument("Expected a number for 'nprobe'")); } } @@ -852,8 +1055,42 @@ zvec::Result ParseHnswRabitqQueryParams( } } - return std::make_shared(ef, radius, is_linear, - is_using_refiner); + auto params = + std::make_shared(nprobe, is_using_refiner); + params->set_radius(radius); + params->set_is_linear(is_linear); + return params; +} + + +zvec::Result ParseDiskAnnQueryParams( + const Napi::Object &obj) { + int list_size{300}; + if (obj.Has("listSize")) { + if (obj.Get("listSize").IsNumber()) { + list_size = obj.Get("listSize").As().Int32Value(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'listSize'")); + } + } + return std::make_shared(list_size); +} + + +zvec::Result ParseFtsQueryParams( + const Napi::Object &obj) { + auto params = std::make_shared(); + if (obj.Has("defaultOperator")) { + if (obj.Get("defaultOperator").IsString()) { + params->set_default_operator( + obj.Get("defaultOperator").As().Utf8Value()); + } else { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a string for 'defaultOperator'")); + } + } + return params; } @@ -956,4 +1193,4 @@ zvec::Result ParseOptimizeOptions( } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/params.h b/src/binding/params.h index 8afea6f..338f3ea 100644 --- a/src/binding/params.h +++ b/src/binding/params.h @@ -2,10 +2,10 @@ #include -#include "zvec/db/doc.h" -#include "zvec/db/index_params.h" -#include "zvec/db/options.h" -#include "zvec/db/status.h" +#include +#include +#include +#include namespace binding { @@ -28,36 +28,49 @@ zvec::Result ParseHnswIndexParams( Napi::Object CreateHnswIndexParams(Napi::Env env, zvec::IndexParams::Ptr params); +zvec::Result ParseHnswRabitqIndexParams( + const Napi::Object &obj); + +Napi::Object CreateHnswRabitqIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params); + zvec::Result ParseIVFIndexParams( const Napi::Object &obj); Napi::Object CreateIVFIndexParams(Napi::Env env, zvec::IndexParams::Ptr params); -zvec::Result ParseHnswRabitqIndexParams( +zvec::Result ParseDiskAnnIndexParams( const Napi::Object &obj); -Napi::Object CreateHnswRabitqIndexParams(Napi::Env env, - zvec::IndexParams::Ptr params); +Napi::Object CreateDiskAnnIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params); zvec::Result ParseInvertIndexParams( const Napi::Object &obj); Napi::Object CreateInvertIndexParams(Napi::Env env, zvec::IndexParams::Ptr params); + +zvec::Result> ParseFtsIndexParams( + const Napi::Object &obj); + +Napi::Object CreateFtsIndexParams(Napi::Env env, zvec::IndexParams::Ptr params); /*** Index Parameters ***/ /*** Query Parameters ***/ -zvec::Result ParseVectorQuery( +zvec::Result ParseSearchQuery( const Napi::Value &value, zvec::CollectionSchema::Ptr schema); zvec::Status ParseVectorToString(const Napi::Value &value, zvec::FieldSchema *schema, - zvec::VectorQuery *query); + zvec::SearchQuery *query); zvec::Status ParseVectorToMap(const Napi::Value &value, zvec::FieldSchema *schema, - zvec::VectorQuery *query); + zvec::SearchQuery *query); + +zvec::Result ParseFtsClause(const Napi::Value &value); zvec::Result ParseQueryParams(const Napi::Value &value); @@ -67,10 +80,16 @@ zvec::Result ParseFlatQueryParams( zvec::Result ParseHnswQueryParams( const Napi::Object &obj); +zvec::Result ParseHnswRabitqQueryParams( + const Napi::Object &obj); + zvec::Result ParseIVFQueryParams( const Napi::Object &obj); -zvec::Result ParseHnswRabitqQueryParams( +zvec::Result ParseDiskAnnQueryParams( + const Napi::Object &obj); + +zvec::Result ParseFtsQueryParams( const Napi::Object &obj); /*** Query Parameters ***/ @@ -96,4 +115,4 @@ zvec::Result ParseOptimizeOptions( /*** Collection-Level Options ***/ -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/zvec b/src/zvec index d9b0920..af88794 160000 --- a/src/zvec +++ b/src/zvec @@ -1 +1 @@ -Subproject commit d9b0920ac7426cad85a79e2daeb652e6698ae2bf +Subproject commit af887944633fc88b3bfa60fc907df893f4cbac83 From 35dba4f78b98023308f0a020753f7ceb7f31c0be Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Mon, 8 Jun 2026 17:53:38 +0800 Subject: [PATCH 02/12] update doc and types --- src/binding/doc.h | 4 ++-- src/binding/params.cc | 1 + src/binding/types.cc | 6 +++++- src/binding/types.h | 6 +++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/binding/doc.h b/src/binding/doc.h index c0de686..f090a18 100644 --- a/src/binding/doc.h +++ b/src/binding/doc.h @@ -2,7 +2,7 @@ #include -#include "zvec/db/doc.h" +#include namespace binding { @@ -63,4 +63,4 @@ Napi::Array CreateScalarArray(Napi::Env &env, const std::string &fieldName, zvec::FieldSchema *schema, zvec::Doc::Ptr &doc); -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/params.cc b/src/binding/params.cc index 8fd01a8..1ff105c 100644 --- a/src/binding/params.cc +++ b/src/binding/params.cc @@ -505,6 +505,7 @@ zvec::Result ParseSearchQuery( } zvec::SearchQuery query{}; + query.topk_ = 10; auto obj = value.As(); zvec::FieldSchema *field_schema{nullptr}; diff --git a/src/binding/types.cc b/src/binding/types.cc index 905dafa..e002f62 100644 --- a/src/binding/types.cc +++ b/src/binding/types.cc @@ -218,7 +218,9 @@ Napi::Object CreateIndexTypeObject(Napi::Env env) { obj.Set("IVF", static_cast(zvec::IndexType::IVF)); obj.Set("FLAT", static_cast(zvec::IndexType::FLAT)); obj.Set("HNSW_RABITQ", static_cast(zvec::IndexType::HNSW_RABITQ)); + obj.Set("DISKANN", static_cast(zvec::IndexType::DISKANN)); obj.Set("INVERT", static_cast(zvec::IndexType::INVERT)); + obj.Set("FTS", static_cast(zvec::IndexType::FTS)); obj.Freeze(); return obj; } @@ -237,7 +239,9 @@ zvec::Result ParseIndexType(const Napi::Value &value) { case zvec::IndexType::IVF: case zvec::IndexType::FLAT: case zvec::IndexType::HNSW_RABITQ: + case zvec::IndexType::DISKANN: case zvec::IndexType::INVERT: + case zvec::IndexType::FTS: return static_cast(raw); default: return tl::make_unexpected(zvec::Status::InvalidArgument( @@ -317,4 +321,4 @@ Napi::Object InitTypes(Napi::Env env, Napi::Object exports) { } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/types.h b/src/binding/types.h index b1dfde9..c7b4956 100644 --- a/src/binding/types.h +++ b/src/binding/types.h @@ -2,8 +2,8 @@ #include -#include "zvec/db/status.h" -#include "zvec/db/type.h" +#include +#include namespace binding { @@ -99,4 +99,4 @@ zvec::Result ParseQuantizeType(const Napi::Value &value); Napi::Object InitTypes(Napi::Env env, Napi::Object exports); -} // namespace binding \ No newline at end of file +} // namespace binding From dda5cc55496a0d55939fbf7b6bfc05b2732a03e9 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Mon, 8 Jun 2026 20:23:38 +0800 Subject: [PATCH 03/12] update headers --- src/binding/async_workers.h | 8 ++--- src/binding/collection.h | 4 +-- src/binding/config.cc | 63 ++++++++++++++++++++++++++++++++----- src/binding/schema.cc | 48 +++++++++++++--------------- src/binding/schema.h | 4 +-- 5 files changed, 85 insertions(+), 42 deletions(-) diff --git a/src/binding/async_workers.h b/src/binding/async_workers.h index 71ad3ad..316579d 100644 --- a/src/binding/async_workers.h +++ b/src/binding/async_workers.h @@ -3,8 +3,8 @@ #include #include -#include "zvec/db/collection.h" -#include "zvec/db/status.h" +#include +#include namespace binding { @@ -30,7 +30,7 @@ class DeleteByFilterWorker : public Napi::AsyncWorker { class QueryWorker : public Napi::AsyncWorker { public: QueryWorker(Napi::Env env, zvec::Collection::Ptr collection, - zvec::CollectionSchema::Ptr schema, zvec::VectorQuery query, + zvec::CollectionSchema::Ptr schema, zvec::SearchQuery query, Napi::Promise::Deferred deferred); void Execute() override; @@ -40,7 +40,7 @@ class QueryWorker : public Napi::AsyncWorker { private: zvec::Collection::Ptr collection_; zvec::CollectionSchema::Ptr schema_; - zvec::VectorQuery query_; + zvec::SearchQuery query_; Napi::Promise::Deferred deferred_; zvec::Status status_; zvec::DocPtrList results_; diff --git a/src/binding/collection.h b/src/binding/collection.h index 1f48f6a..e317e52 100644 --- a/src/binding/collection.h +++ b/src/binding/collection.h @@ -2,7 +2,7 @@ #include -#include "zvec/db/collection.h" +#include #include "addon.h" @@ -85,4 +85,4 @@ class Collection : public Napi::ObjectWrap { }; -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/config.cc b/src/binding/config.cc index d92ed4c..9586f9a 100644 --- a/src/binding/config.cc +++ b/src/binding/config.cc @@ -1,5 +1,5 @@ #include "config.h" -#include "zvec/db/config.h" +#include #include "types.h" @@ -17,11 +17,11 @@ Napi::Object CreateLogTypeObject(Napi::Env env) { Napi::Object CreateLogLevelObject(Napi::Env env) { Napi::Object obj = Napi::Object::New(env); - obj.Set("DEBUG", static_cast(zvec::GlobalConfig::LogLevel::DEBUG)); - obj.Set("INFO", static_cast(zvec::GlobalConfig::LogLevel::INFO)); - obj.Set("WARN", static_cast(zvec::GlobalConfig::LogLevel::WARN)); - obj.Set("ERROR", static_cast(zvec::GlobalConfig::LogLevel::ERROR)); - obj.Set("FATAL", static_cast(zvec::GlobalConfig::LogLevel::FATAL)); + obj.Set("DEBUG", static_cast(zvec::GlobalConfig::LogLevel::kDebug)); + obj.Set("INFO", static_cast(zvec::GlobalConfig::LogLevel::kInfo)); + obj.Set("WARN", static_cast(zvec::GlobalConfig::LogLevel::kWarn)); + obj.Set("ERROR", static_cast(zvec::GlobalConfig::LogLevel::kError)); + obj.Set("FATAL", static_cast(zvec::GlobalConfig::LogLevel::kFatal)); obj.Freeze(); return obj; } @@ -41,7 +41,7 @@ Napi::Value Initialize(const Napi::CallbackInfo &info) { if (obj.Has("logType") || obj.Has("logLevel")) { std::string log_type = "console"; - zvec::GlobalConfig::LogLevel log_level{zvec::GlobalConfig::LogLevel::WARN}; + zvec::GlobalConfig::LogLevel log_level{zvec::GlobalConfig::LogLevel::kWarn}; if (obj.Has("logType")) { if (obj.Get("logType").IsString()) { log_type = obj.Get("logType").As().Utf8Value(); @@ -160,6 +160,29 @@ Napi::Value Initialize(const Napi::CallbackInfo &info) { } } + if (obj.Has("ftsBruteForceByKeysRatio")) { + if (obj.Get("ftsBruteForceByKeysRatio").IsNumber()) { + config.fts_brute_force_by_keys_ratio = + obj.Get("ftsBruteForceByKeysRatio").As().FloatValue(); + } else { + ThrowIfNotOk(env, + zvec::Status::InvalidArgument("Expected a number for " + "'ftsBruteForceByKeysRatio'")); + return env.Undefined(); + } + } + + if (obj.Has("jiebaDictDir")) { + if (obj.Get("jiebaDictDir").IsString()) { + config.jieba_dict_dir = + obj.Get("jiebaDictDir").As().Utf8Value(); + } else { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Expected a string for 'jiebaDictDir'")); + return env.Undefined(); + } + } + if (auto s = zvec::GlobalConfig::Instance().Initialize(config); !s.ok()) { ThrowIfNotOk(env, s); } @@ -167,12 +190,36 @@ Napi::Value Initialize(const Napi::CallbackInfo &info) { } +Napi::Value SetDefaultJiebaDictDir(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1 || !info[0].IsString()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "setDefaultJiebaDictDir() expects exactly one " + "argument: (dir: string)")); + return env.Undefined(); + } + zvec::GlobalConfig::Instance().set_default_jieba_dict_dir( + info[0].As().Utf8Value()); + return env.Undefined(); +} + + +Napi::Value GetDefaultJiebaDictDir(const Napi::CallbackInfo &info) { + return Napi::String::New(info.Env(), + zvec::GlobalConfig::Instance().jieba_dict_dir()); +} + + Napi::Object InitConfig(Napi::Env env, Napi::Object exports) { exports.Set("LogType", CreateLogTypeObject(env)); exports.Set("LogLevel", CreateLogLevelObject(env)); exports.Set("initialize", Napi::Function::New(env, Initialize)); + exports.Set("setDefaultJiebaDictDir", + Napi::Function::New(env, SetDefaultJiebaDictDir)); + exports.Set("getDefaultJiebaDictDir", + Napi::Function::New(env, GetDefaultJiebaDictDir)); return exports; } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/schema.cc b/src/binding/schema.cc index fd14e2c..df656bb 100644 --- a/src/binding/schema.cc +++ b/src/binding/schema.cc @@ -182,8 +182,8 @@ CollectionSchema::CollectionSchema(const Napi::CallbackInfo &info) if (info.Length() != 1) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "CollectionSchema constructor expects 1 argument " - "(schema definition object): { name: string, fields: " - "[...], vectors: [...] }")); + "(schema definition object): { name: string, " + "fields?: [...], vectors?: [...] }")); return; } @@ -219,35 +219,31 @@ CollectionSchema::CollectionSchema(const Napi::CallbackInfo &info) std::vector fields; - if (!obj.Has("vectors")) { - ThrowIfNotOk( - env, zvec::Status::InvalidArgument( - "Missing required argument 'vectors' in CollectionSchema")); - return; - } - Napi::Value vectorsValue = obj.Get("vectors"); - if (vectorsValue.IsArray()) { - Napi::Array vectorsArray = vectorsValue.As(); - for (uint32_t i = 0; i < vectorsArray.Length(); i++) { - auto parsed_vector_schema = ParseVectorSchema(vectorsArray.Get(i)); + if (obj.Has("vectors")) { + Napi::Value vectorsValue = obj.Get("vectors"); + if (vectorsValue.IsArray()) { + Napi::Array vectorsArray = vectorsValue.As(); + for (uint32_t i = 0; i < vectorsArray.Length(); i++) { + auto parsed_vector_schema = ParseVectorSchema(vectorsArray.Get(i)); + if (!parsed_vector_schema) { + ThrowIfNotOk(env, parsed_vector_schema.error()); + return; + } + fields.emplace_back(parsed_vector_schema.value()); + } + } else if (vectorsValue.IsObject()) { + auto parsed_vector_schema = ParseVectorSchema(vectorsValue); if (!parsed_vector_schema) { ThrowIfNotOk(env, parsed_vector_schema.error()); return; } fields.emplace_back(parsed_vector_schema.value()); - } - } else if (vectorsValue.IsObject()) { - auto parsed_vector_schema = ParseVectorSchema(vectorsValue); - if (!parsed_vector_schema) { - ThrowIfNotOk(env, parsed_vector_schema.error()); + } else if (!vectorsValue.IsUndefined() && !vectorsValue.IsNull()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "CollectionSchema constructor: argument 'vectors' " + "must be an object or an array of objects")); return; } - fields.emplace_back(parsed_vector_schema.value()); - } else { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "CollectionSchema constructor: argument 'vectors' " - "must be an object or an array of objects")); - return; } if (obj.Has("fields")) { @@ -269,7 +265,7 @@ CollectionSchema::CollectionSchema(const Napi::CallbackInfo &info) return; } fields.emplace_back(parsed_field_schema.value()); - } else { + } else if (!fieldsValue.IsUndefined() && !fieldsValue.IsNull()) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "CollectionSchema constructor: argument 'fields' " "must be an object or an array of objects")); @@ -387,4 +383,4 @@ Napi::Value CollectionSchema::ToString(const Napi::CallbackInfo &info) { } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/schema.h b/src/binding/schema.h index 87f92b0..6666b79 100644 --- a/src/binding/schema.h +++ b/src/binding/schema.h @@ -2,7 +2,7 @@ #include -#include "zvec/db/schema.h" +#include #include "addon.h" @@ -49,4 +49,4 @@ class CollectionSchema : public Napi::ObjectWrap { }; -} // namespace binding \ No newline at end of file +} // namespace binding From 499d491d3c487b1a788415559e22fc4921332551 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 14:01:04 +0800 Subject: [PATCH 04/12] update collections --- src/binding/async_workers.cc | 2 +- src/binding/collection.cc | 84 ++++++++++++++++++++++++++++-------- src/zvec | 2 +- 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/src/binding/async_workers.cc b/src/binding/async_workers.cc index 4e0f9d1..7ad65e8 100644 --- a/src/binding/async_workers.cc +++ b/src/binding/async_workers.cc @@ -31,7 +31,7 @@ void DeleteByFilterWorker::OnError(const Napi::Error &error) { QueryWorker::QueryWorker(Napi::Env env, zvec::Collection::Ptr collection, zvec::CollectionSchema::Ptr schema, - zvec::VectorQuery query, + zvec::SearchQuery query, Napi::Promise::Deferred deferred) : Napi::AsyncWorker(env), collection_(collection), diff --git a/src/binding/collection.cc b/src/binding/collection.cc index ebef24a..3331983 100644 --- a/src/binding/collection.cc +++ b/src/binding/collection.cc @@ -1,4 +1,5 @@ #include "collection.h" +#include #include "async_workers.h" #include "doc.h" #include "params.h" @@ -570,7 +571,7 @@ Napi::Value Collection::Query(const Napi::CallbackInfo &info) { return env.Undefined(); } - if (auto parsed_query = ParseVectorQuery(info[0], get_wrapped_schema()); + if (auto parsed_query = ParseSearchQuery(info[0], get_wrapped_schema()); parsed_query) { auto res = collection_->Query(parsed_query.value()); if (res) { @@ -604,7 +605,7 @@ Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { return deferred.Promise(); } - if (auto parsed_query = ParseVectorQuery(info[0], get_wrapped_schema()); + if (auto parsed_query = ParseSearchQuery(info[0], get_wrapped_schema()); parsed_query) { auto *worker = new QueryWorker(env, collection_, get_wrapped_schema(), std::move(parsed_query.value()), deferred); @@ -620,42 +621,87 @@ Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { Napi::Value Collection::Fetch(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (ThrowIfClosed(env)) return env.Undefined(); - if (info.Length() != 1) { - ThrowIfNotOk( - env, zvec::Status::InvalidArgument( - "Collection.fetch(): Expected exactly 1 argument. " - "Argument must be an id string or an array of id strings.")); + if (info.Length() != 1 || !info[0].IsObject()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected exactly 1 argument: " + "({ ids: string | string[], outputFields?: string[], " + "includeVector?: boolean }).")); return env.Undefined(); } - std::vector pks{}; zvec::Result res; + std::vector pks{}; + std::optional> output_fields{}; + bool include_vector{true}; + auto obj = info[0].As(); - if (info[0].IsArray()) { - Napi::Array pkArray = info[0].As(); + if (!obj.Has("ids")) { + ThrowIfNotOk(env, + zvec::Status::InvalidArgument( + "Collection.fetch(): Missing required argument 'ids'")); + return env.Undefined(); + } + if (obj.Has("outputFields") && !obj.Get("outputFields").IsUndefined()) { + if (!obj.Get("outputFields").IsArray()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): argument 'outputFields' must " + "be an array of strings")); + return env.Undefined(); + } + std::vector fields{}; + auto array = obj.Get("outputFields").As(); + fields.reserve(array.Length()); + for (uint32_t i = 0; i < array.Length(); i++) { + auto item = array.Get(i); + if (!item.IsString()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected an array of " + "strings for 'outputFields'")); + return env.Undefined(); + } + fields.emplace_back(item.As().Utf8Value()); + } + output_fields = std::move(fields); + } + if (obj.Has("includeVector") && !obj.Get("includeVector").IsUndefined()) { + if (!obj.Get("includeVector").IsBoolean()) { + ThrowIfNotOk( + env, + zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a boolean for 'includeVector'")); + return env.Undefined(); + } + include_vector = obj.Get("includeVector").As().Value(); + } + + auto ids = obj.Get("ids"); + if (ids.IsArray()) { + Napi::Array pkArray = ids.As(); uint32_t length = pkArray.Length(); pks.reserve(length); for (uint32_t i = 0; i < length; i++) { if (pkArray.Get(i).IsString()) { pks.emplace_back(pkArray.Get(i).As().Utf8Value()); } else { - ThrowIfNotOk(env, - zvec::Status::InvalidArgument("Expected a string for id")); + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a string or array " + "of strings for 'ids'")); return env.Undefined(); } } - res = collection_->Fetch(pks); } else { - if (info[0].IsString()) { - pks.emplace_back(info[0].As().Utf8Value()); + if (ids.IsString()) { + pks.emplace_back(ids.As().Utf8Value()); } else { - ThrowIfNotOk(env, - zvec::Status::InvalidArgument("Expected a string for id")); + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a string or array of " + "strings for 'ids'")); return env.Undefined(); } - res = collection_->Fetch(pks); } + res = collection_->Fetch(pks, output_fields, include_vector); + if (res) { Napi::Object obj = Napi::Object::New(env); const zvec::DocPtrMap &map = res.value(); @@ -1006,4 +1052,4 @@ Napi::Value Collection::DropIndex(const Napi::CallbackInfo &info) { } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/zvec b/src/zvec index af88794..0923f7c 160000 --- a/src/zvec +++ b/src/zvec @@ -1 +1 @@ -Subproject commit af887944633fc88b3bfa60fc907df893f4cbac83 +Subproject commit 0923f7c6910d46907420e08f6225c82395dab9d3 From d2f61ac115455f5c0494546521ef934af7b1e9bf Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 14:44:55 +0800 Subject: [PATCH 05/12] update build script --- CMakeLists.txt | 15 +++++++++++++++ package.json | 3 ++- packages/bindings-linux-x64/package.json | 5 +++-- scripts/build.js | 12 +++++++++++- scripts/pack-local.js | 11 ++++++++++- src/binding/addon.cc | 2 +- 6 files changed, 42 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b9cc5a0..87b7d5e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,10 @@ set(ZVEC_BUILD_TARGETS --target zvec_turbo --target zvec_db ) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64") + list(APPEND ZVEC_BUILD_TARGETS --target core_knn_diskann) + list(APPEND ZVEC_BYPRODUCTS ${ZVEC_LIB_DIR}/libzvec_diskann_plugin.so) +endif() set(ZVEC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release) if(MSVC) list(APPEND ZVEC_CMAKE_ARGS @@ -141,6 +145,7 @@ if (NOT WIN32) ${GLOG_LIB} ${GFLAGS_LIB} ${PROTOBUF_LIB} + FastPFOR lz4 ) else () @@ -165,6 +170,7 @@ else () ${GLOG_LIB} ${GFLAGS_LIB} ${PROTOBUF_LIB} + FastPFOR lz4 rpcrt4 shlwapi @@ -318,3 +324,12 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release") ) endif() endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64") + add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${ZVEC_LIB_DIR}/libzvec_diskann_plugin.so" + "$/libzvec_diskann_plugin.so" + COMMENT "Copying DiskANN runtime plugin" + ) +endif() diff --git a/package.json b/package.json index ba9be48..09b3451 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "src/index.d.mts", "scripts/install.js", "zvec_node_binding.node", + "libzvec_diskann_plugin.so", "README.md", "LICENSE" ], @@ -79,4 +80,4 @@ "dependencies": { "bindings": "^1.5.0" } -} +} \ No newline at end of file diff --git a/packages/bindings-linux-x64/package.json b/packages/bindings-linux-x64/package.json index 0bbe667..393f869 100644 --- a/packages/bindings-linux-x64/package.json +++ b/packages/bindings-linux-x64/package.json @@ -10,6 +10,7 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node" + "zvec_node_binding.node", + "libzvec_diskann_plugin.so" ] -} +} \ No newline at end of file diff --git a/scripts/build.js b/scripts/build.js index f9e120d..96590bf 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -56,8 +56,18 @@ try { const targetPath = path.join(platformPackageDir, 'zvec_node_binding.node'); fs.copyFileSync(binaryPath, targetPath); + // DiskANN currently ships as a runtime-loaded plugin. Keep it next to the + // addon binary so the C++ engine can auto-load it when DiskANN is used. + const diskAnnPluginPath = path.join(BUILD_TARGET_DIR, 'libzvec_diskann_plugin.so'); + if (fs.existsSync(diskAnnPluginPath)) { + fs.copyFileSync( + diskAnnPluginPath, + path.join(platformPackageDir, 'libzvec_diskann_plugin.so') + ); + } + console.log(`✅ Binary compiled and packaged for ${platformPackageName} at: ${targetPath}`); } catch (error) { console.error('❌ Error during build and packaging:', error.message); process.exit(1); -} \ No newline at end of file +} diff --git a/scripts/pack-local.js b/scripts/pack-local.js index aed3201..5c30bcb 100644 --- a/scripts/pack-local.js +++ b/scripts/pack-local.js @@ -26,6 +26,15 @@ try { fs.copyFileSync(targetPath, destPath); console.log(`Binary copied from ${targetPath} to ${destPath}`); + // DiskANN currently ships as a runtime-loaded plugin. Local packs bundle it + // beside the addon binary so the C++ engine can auto-load it when needed. + const pluginPath = path.join(platformPackageDir, 'libzvec_diskann_plugin.so'); + if (fs.existsSync(pluginPath)) { + const pluginDestPath = path.join(PACKAGE_ROOT, 'libzvec_diskann_plugin.so'); + fs.copyFileSync(pluginPath, pluginDestPath); + console.log(`DiskANN plugin copied from ${pluginPath} to ${pluginDestPath}`); + } + // Temporarily remove optionalDependencies from package.json // (local pack bundles the binary directly, no need for platform packages) const packageJsonPath = path.join(PACKAGE_ROOT, 'package.json'); @@ -44,4 +53,4 @@ try { } catch (error) { console.error('❌ Error during build and packaging:', error.message); process.exit(1); -} \ No newline at end of file +} diff --git a/src/binding/addon.cc b/src/binding/addon.cc index 2d4569c..d08c0e4 100644 --- a/src/binding/addon.cc +++ b/src/binding/addon.cc @@ -39,4 +39,4 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { } -NODE_API_MODULE(zvec_node_binding, Init) \ No newline at end of file +NODE_API_MODULE(zvec_node_binding, Init) From 7e9f20e249ea5a78691d2ae209da807ff8d995ee Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 16:07:03 +0800 Subject: [PATCH 06/12] update --- src/index.d.mts | 24 +++-- src/index.d.ts | 256 +++++++++++++++++++++++++++++++++++++++++++----- src/index.js | 4 +- src/index.mjs | 2 + 4 files changed, 256 insertions(+), 30 deletions(-) diff --git a/src/index.d.mts b/src/index.d.mts index 728e108..e02e1f8 100644 --- a/src/index.d.mts +++ b/src/index.d.mts @@ -1,14 +1,17 @@ export { - isZVecError, ZVecCollectionSchema, + isZVecError, + ZVecCollectionSchema, ZVecCreateAndOpen, ZVecDataType, + ZVecGetDefaultJiebaDictDir, ZVecIndexType, ZVecInitialize, ZVecLogLevel, ZVecLogType, ZVecMetricType, ZVecOpen, - ZVecQuantizeType + ZVecQuantizeType, + ZVecSetDefaultJiebaDictDir } from './index.js'; export type { @@ -17,18 +20,26 @@ export type { ZVecCollection, ZVecCollectionOptions, ZVecCreateIndexOptions, + ZVecDiskAnnIndexParams, + ZVecDiskAnnQueryParams, ZVecDoc, ZVecDocInput, ZVecError, ZVecFieldSchema, ZVecFlatIndexParams, + ZVecFtsIndexParams, + ZVecFtsQuery, + ZVecFtsQueryParams, ZVecHnswIndexParams, ZVecHnswQueryParams, ZVecHnswRabitqIndexParams, - ZVecHnswRabitqQueryParams, ZVecIndexParams, + ZVecHnswRabitqQueryParams, + ZVecIndexParams, ZVecInitOptions, - ZVecInvertIndexParams, ZVecIVFIndexParams, - ZVecIVFQueryParams, ZVecOptimizeOptions, + ZVecInvertIndexParams, + ZVecIVFIndexParams, + ZVecIVFQueryParams, + ZVecOptimizeOptions, ZVecQuery, ZVecQueryParams, ZVecStatus, @@ -45,9 +56,10 @@ declare const _default: { ZVecLogLevel: typeof import('./index.js').ZVecLogLevel; ZVecCollectionSchema: typeof import('./index.js').ZVecCollectionSchema; ZVecInitialize: typeof import('./index.js').ZVecInitialize; + ZVecSetDefaultJiebaDictDir: typeof import('./index.js').ZVecSetDefaultJiebaDictDir; + ZVecGetDefaultJiebaDictDir: typeof import('./index.js').ZVecGetDefaultJiebaDictDir; ZVecCreateAndOpen: typeof import('./index.js').ZVecCreateAndOpen; ZVecOpen: typeof import('./index.js').ZVecOpen; isZVecError: typeof import('./index.js').isZVecError; }; export default _default; - diff --git a/src/index.d.ts b/src/index.d.ts index 4e0c9e8..c770e18 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -52,7 +52,9 @@ export declare const ZVecIndexType: { readonly IVF: 2; readonly FLAT: 3; readonly HNSW_RABITQ: 4; + readonly DISKANN: 5; readonly INVERT: 10; + readonly FTS: 11; }; export type ZVecIndexType = typeof ZVecIndexType[keyof typeof ZVecIndexType]; @@ -181,18 +183,55 @@ export interface ZVecInitOptions { * @default undefined // Let the system decide */ readonly optimizeThreads?: number; + + /** + * Threshold to switch from inverted-index filtering to forward scan. + * @default undefined + */ + readonly invertToForwardScanRatio?: number; + + /** + * Threshold to use brute-force key lookup over index lookup. + * @default undefined + */ + readonly bruteForceByKeysRatio?: number; + + /** + * Threshold to switch FTS scan strategy for highly selective candidates. + * @default undefined + */ + readonly ftsBruteForceByKeysRatio?: number; + + /** + * Directory containing jieba tokenizer dictionary files. + * @default undefined + */ + readonly jiebaDictDir?: string; } /** * Initializes global Zvec configurations. * This function should be called only once at the start of your application. - * @param options - Optional configuration parameters. + * @param options - Configuration parameters. * * @group Global Configuration */ -export function ZVecInitialize(options?: ZVecInitOptions): void; +export function ZVecInitialize(options: ZVecInitOptions): void; +/** + * Registers the process-wide default jieba dictionary directory. + * + * @group Global Configuration + */ +export function ZVecSetDefaultJiebaDictDir(dir: string): void; + +/** + * Reads the process-wide default jieba dictionary directory. + * + * @group Global Configuration + */ +export function ZVecGetDefaultJiebaDictDir(): string; /** * Base interface for index parameters, requiring the type of index. @@ -259,6 +298,12 @@ export interface ZVecHnswIndexParams extends ZVecIndexParams { * @default ZVecQuantizeType.UNDEFINED */ readonly quantizeType?: ZVecQuantizeType; + + /** + * Whether to allocate HNSW graph nodes in contiguous memory. + * @default false + */ + readonly useContiguousMemory?: boolean; } @@ -349,6 +394,47 @@ export interface ZVecIVFIndexParams extends ZVecIndexParams { } +/** + * Configuration parameters for a DiskANN index on a vector field. + * + * @group Index Parameters + */ +export interface ZVecDiskAnnIndexParams extends ZVecIndexParams { + /** Must be `ZVecIndexType.DISKANN` (5). */ + readonly indexType: typeof ZVecIndexType.DISKANN; + + /** + * Distance metric used for similarity computation. + * @default ZVecMetricType.IP + */ + readonly metricType?: ZVecMetricType; + + /** + * Maximum out-degree of each graph node. + * @default 100 + */ + readonly maxDegree?: number; + + /** + * Candidate list size used during graph construction. + * @default 50 + */ + readonly listSize?: number; + + /** + * Maximum number of PQ chunks for product quantization. `0` lets the engine choose. + * @default 0 + */ + readonly pqChunkNum?: number; + + /** + * Optional quantization type for vector compression. + * @default ZVecQuantizeType.UNDEFINED + */ + readonly quantizeType?: ZVecQuantizeType; +} + + /** * Configuration parameters for an inverted index on a scalar field. * @@ -372,6 +458,35 @@ export interface ZVecInvertIndexParams extends ZVecIndexParams { } +/** + * Configuration parameters for a full-text search index on a scalar string field. + * + * @group Index Parameters + */ +export interface ZVecFtsIndexParams extends ZVecIndexParams { + /** Must be `ZVecIndexType.FTS` (11). */ + readonly indexType: typeof ZVecIndexType.FTS; + + /** + * Tokenizer name. + * @default "standard" + */ + readonly tokenizerName?: string; + + /** + * Token filters. + * @default ["lowercase"] + */ + readonly filters?: string[]; + + /** + * Additional tokenizer parameters as a JSON object string. + * @default "" + */ + readonly extraParams?: string; +} + + /** * Base interface for query-time parameters, requiring the type of index. * @@ -379,7 +494,15 @@ export interface ZVecInvertIndexParams extends ZVecIndexParams { */ export interface ZVecQueryParams { readonly indexType: ZVecIndexType; +} + +/** + * Base interface for vector query-time parameters. + * + * @group Query Parameters + */ +interface ZVecVectorQueryParams extends ZVecQueryParams { /** * Search radius for range queries. Used in combination with top-k to filter results. * @default 0.0 (disabled) @@ -405,7 +528,7 @@ export interface ZVecQueryParams { * * @group Query Parameters */ -export interface ZVecHnswQueryParams extends ZVecQueryParams { +export interface ZVecHnswQueryParams extends ZVecVectorQueryParams { /** Must be `ZVecIndexType.HNSW` (1). */ readonly indexType: typeof ZVecIndexType.HNSW; @@ -424,7 +547,7 @@ export interface ZVecHnswQueryParams extends ZVecQueryParams { * * @group Query Parameters */ -export interface ZVecHnswRabitqQueryParams extends ZVecQueryParams { +export interface ZVecHnswRabitqQueryParams extends ZVecVectorQueryParams { /** Must be `ZVecIndexType.HNSW_RABITQ` (4). */ readonly indexType: typeof ZVecIndexType.HNSW_RABITQ; @@ -441,7 +564,7 @@ export interface ZVecHnswRabitqQueryParams extends ZVecQueryParams { * * @group Query Parameters */ -export interface ZVecIVFQueryParams extends ZVecQueryParams { +export interface ZVecIVFQueryParams extends ZVecVectorQueryParams { /** Must be `ZVecIndexType.IVF` (2). */ readonly indexType: typeof ZVecIndexType.IVF; @@ -453,11 +576,66 @@ export interface ZVecIVFQueryParams extends ZVecQueryParams { } +/** + * Query-time parameters for searches performed against a DiskANN index. + * + * @group Query Parameters + */ +export interface ZVecDiskAnnQueryParams extends ZVecVectorQueryParams { + /** Must be `ZVecIndexType.DISKANN` (5). */ + readonly indexType: typeof ZVecIndexType.DISKANN; + + /** + * Candidate list size used during search. Larger values improve recall but slow down search. + * @default 300 + */ + readonly listSize?: number; +} + + +/** + * Query-time parameters for full-text search. + * + * @group Query Parameters + */ +export interface ZVecFtsQueryParams extends ZVecQueryParams { + /** Must be `ZVecIndexType.FTS` (11). */ + readonly indexType: typeof ZVecIndexType.FTS; + + /** + * Default boolean operator for adjacent bare terms. Supported values are "AND" and "OR"; empty string uses OR. + * @default "" + */ + readonly defaultOperator?: string; +} + + +/** + * Full-text search clause. Provide exactly one of `queryString` or `matchString`. + * + * @group Query Parameters + */ +export interface ZVecFtsQuery { + /** + * Query parser expression. + * @default undefined + */ + readonly queryString?: string; + + /** + * Literal match string. + * @default undefined + */ + readonly matchString?: string; +} + + /** * Query object used to perform searches against a collection. * * You can use it for: * - Vector similarity search: provide both `fieldName` and `vector`. + * - Full-text search: provide both `fieldName` and `fts`. * - Scalar-only filtering: provide only `filter`. * - Hybrid search: provide `fieldName` + `vector` along with a scalar `filter`. * @@ -481,6 +659,12 @@ export interface ZVecQuery { */ vector?: ZVecVector; + /** + * Full-text search clause. Cannot be combined with `vector` in the same query. + * @default undefined + */ + fts?: ZVecFtsQuery; + /** * Boolean expression to pre-filter candidates. * @default undefined @@ -503,7 +687,12 @@ export interface ZVecQuery { * Query-time parameters to fine-tune search behavior. * @default undefined */ - params?: ZVecHnswQueryParams | ZVecHnswRabitqQueryParams | ZVecIVFQueryParams; + params?: + | ZVecHnswQueryParams + | ZVecHnswRabitqQueryParams + | ZVecIVFQueryParams + | ZVecDiskAnnQueryParams + | ZVecFtsQueryParams; } @@ -667,10 +856,10 @@ export interface ZVecFieldSchema { readonly nullable?: boolean; /** - * Optional inverted index parameters for this field. + * Optional scalar index parameters for this field. * @default undefined */ - readonly indexParams?: ZVecInvertIndexParams; + readonly indexParams?: ZVecInvertIndexParams | ZVecFtsIndexParams; } @@ -700,7 +889,12 @@ export interface ZVecVectorSchema { * Vector index parameters for this vector field. * @default undefined */ - readonly indexParams?: ZVecFlatIndexParams | ZVecHnswIndexParams | ZVecHnswRabitqIndexParams | ZVecIVFIndexParams; + readonly indexParams?: + | ZVecFlatIndexParams + | ZVecHnswIndexParams + | ZVecHnswRabitqIndexParams + | ZVecIVFIndexParams + | ZVecDiskAnnIndexParams; } @@ -712,14 +906,14 @@ export interface ZVecVectorSchema { export declare class ZVecCollectionSchema { /** * Creates a new collection schema. - * @param params - An object containing the name, vectors, and optional fields. + * @param params - An object containing the collection name, vector fields, and scalar fields. * @param params.name - The name of the collection. - * @param params.vectors - An array of vector schemas, or a single vector schema object. + * @param params.vectors - An optional array of vector schemas, or a single vector schema object. * @param params.fields - An optional array of scalar field schemas, or a single scalar field schema object. */ constructor(params: { name: string; - vectors: ZVecVectorSchema[] | ZVecVectorSchema; + vectors?: ZVecVectorSchema[] | ZVecVectorSchema; fields?: ZVecFieldSchema[] | ZVecFieldSchema; }); @@ -932,10 +1126,17 @@ export declare class ZVecCollection { /** * Fetches documents by their IDs. - * @param ids - A single document ID or an array of document IDs to fetch. + * @param params - Fetch parameters. + * @param params.ids - A single document ID or an array of document IDs to fetch. + * @param params.outputFields - Scalar fields to include. If undefined, all scalar fields are returned. An empty array returns no scalar fields. + * @param params.includeVector - Whether to include vector data in fetched documents. Defaults to true. * @returns An object mapping the requested IDs to their corresponding documents. If an ID is not found, it will not be present in the returned object. */ - fetchSync(ids: string | string[]): Record; + fetchSync(params: { + ids: string | string[]; + outputFields?: string[]; + includeVector?: boolean; + }): Record; /** * Optimizes the collection's internal structures for better performance. @@ -1015,7 +1216,14 @@ export declare class ZVecCollection { */ createIndexSync(params: { fieldName: string; - indexParams: ZVecFlatIndexParams | ZVecHnswIndexParams | ZVecHnswRabitqIndexParams | ZVecIVFIndexParams | ZVecInvertIndexParams; + indexParams: + | ZVecFlatIndexParams + | ZVecHnswIndexParams + | ZVecHnswRabitqIndexParams + | ZVecIVFIndexParams + | ZVecDiskAnnIndexParams + | ZVecInvertIndexParams + | ZVecFtsIndexParams; indexOptions?: ZVecCreateIndexOptions; }): void; @@ -1045,13 +1253,13 @@ export function ZVecCreateAndOpen( /** - * Opens an existing Zvec collection. - * @param path - The file system path where the collection is stored. - * @param options - Optional parameters for opening the collection. - * @returns An instance of ZVecCollection. - * - * @group Collection - */ + * Opens an existing Zvec collection. + * @param path - The file system path where the collection is stored. + * @param options - Optional parameters for opening the collection. + * @returns An instance of ZVecCollection. + * + * @group Collection + */ export function ZVecOpen( path: string, options?: ZVecCollectionOptions @@ -1067,8 +1275,10 @@ declare const _default: { ZVecLogLevel: typeof ZVecLogLevel; ZVecCollectionSchema: typeof ZVecCollectionSchema; ZVecInitialize: typeof ZVecInitialize; + ZVecSetDefaultJiebaDictDir: typeof ZVecSetDefaultJiebaDictDir; + ZVecGetDefaultJiebaDictDir: typeof ZVecGetDefaultJiebaDictDir; ZVecCreateAndOpen: typeof ZVecCreateAndOpen; ZVecOpen: typeof ZVecOpen; isZVecError: typeof isZVecError; }; -export default _default; \ No newline at end of file +export default _default; diff --git a/src/index.js b/src/index.js index 80d54db..ec9d54a 100644 --- a/src/index.js +++ b/src/index.js @@ -78,7 +78,9 @@ module.exports = { ZVecLogLevel: binding.LogLevel, ZVecCollectionSchema: binding.CollectionSchema, ZVecInitialize: binding.initialize, + ZVecSetDefaultJiebaDictDir: binding.setDefaultJiebaDictDir, + ZVecGetDefaultJiebaDictDir: binding.getDefaultJiebaDictDir, ZVecCreateAndOpen: binding.createAndOpen, ZVecOpen: binding.open, isZVecError, -}; \ No newline at end of file +}; diff --git a/src/index.mjs b/src/index.mjs index 348814a..e0e09dd 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -11,6 +11,8 @@ export const ZVecLogType = cjs.ZVecLogType; export const ZVecLogLevel = cjs.ZVecLogLevel; export const ZVecCollectionSchema = cjs.ZVecCollectionSchema; export const ZVecInitialize = cjs.ZVecInitialize; +export const ZVecSetDefaultJiebaDictDir = cjs.ZVecSetDefaultJiebaDictDir; +export const ZVecGetDefaultJiebaDictDir = cjs.ZVecGetDefaultJiebaDictDir; export const ZVecCreateAndOpen = cjs.ZVecCreateAndOpen; export const ZVecOpen = cjs.ZVecOpen; export const isZVecError = cjs.isZVecError; From 89d9d07d99aeba9266ee7c5c4efab840616b6583 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 16:49:24 +0800 Subject: [PATCH 07/12] update fetch --- src/binding/collection.cc | 121 ++++++++++++++++++++------------------ src/index.d.ts | 7 +++ 2 files changed, 72 insertions(+), 56 deletions(-) diff --git a/src/binding/collection.cc b/src/binding/collection.cc index 3331983..1284f12 100644 --- a/src/binding/collection.cc +++ b/src/binding/collection.cc @@ -621,9 +621,11 @@ Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { Napi::Value Collection::Fetch(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (ThrowIfClosed(env)) return env.Undefined(); - if (info.Length() != 1 || !info[0].IsObject()) { + if (info.Length() != 1 || + !(info[0].IsObject() || info[0].IsString() || info[0].IsArray())) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "Collection.fetch(): Expected exactly 1 argument: " + "(ids: string | string[]) or " "({ ids: string | string[], outputFields?: string[], " "includeVector?: boolean }).")); return env.Undefined(); @@ -633,71 +635,78 @@ Napi::Value Collection::Fetch(const Napi::CallbackInfo &info) { std::vector pks{}; std::optional> output_fields{}; bool include_vector{true}; - auto obj = info[0].As(); - if (!obj.Has("ids")) { - ThrowIfNotOk(env, - zvec::Status::InvalidArgument( - "Collection.fetch(): Missing required argument 'ids'")); - return env.Undefined(); - } - if (obj.Has("outputFields") && !obj.Get("outputFields").IsUndefined()) { - if (!obj.Get("outputFields").IsArray()) { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "Collection.fetch(): argument 'outputFields' must " - "be an array of strings")); - return env.Undefined(); - } - std::vector fields{}; - auto array = obj.Get("outputFields").As(); - fields.reserve(array.Length()); - for (uint32_t i = 0; i < array.Length(); i++) { - auto item = array.Get(i); - if (!item.IsString()) { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "Collection.fetch(): Expected an array of " - "strings for 'outputFields'")); - return env.Undefined(); + auto parse_ids = [&](const Napi::Value &ids) -> bool { + if (ids.IsArray()) { + Napi::Array pkArray = ids.As(); + uint32_t length = pkArray.Length(); + pks.reserve(length); + for (uint32_t i = 0; i < length; i++) { + if (pkArray.Get(i).IsString()) { + pks.emplace_back(pkArray.Get(i).As().Utf8Value()); + } else { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a string or array " + "of strings for 'ids'")); + return false; + } } - fields.emplace_back(item.As().Utf8Value()); + return true; } - output_fields = std::move(fields); - } - if (obj.Has("includeVector") && !obj.Get("includeVector").IsUndefined()) { - if (!obj.Get("includeVector").IsBoolean()) { - ThrowIfNotOk( - env, - zvec::Status::InvalidArgument( - "Collection.fetch(): Expected a boolean for 'includeVector'")); - return env.Undefined(); + if (ids.IsString()) { + pks.emplace_back(ids.As().Utf8Value()); + return true; } - include_vector = obj.Get("includeVector").As().Value(); - } + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a string or array of " + "strings for 'ids'")); + return false; + }; - auto ids = obj.Get("ids"); - if (ids.IsArray()) { - Napi::Array pkArray = ids.As(); - uint32_t length = pkArray.Length(); - pks.reserve(length); - for (uint32_t i = 0; i < length; i++) { - if (pkArray.Get(i).IsString()) { - pks.emplace_back(pkArray.Get(i).As().Utf8Value()); - } else { + if (info[0].IsString() || info[0].IsArray()) { + if (!parse_ids(info[0])) return env.Undefined(); + } else { + auto obj = info[0].As(); + if (!obj.Has("ids")) { + ThrowIfNotOk(env, + zvec::Status::InvalidArgument( + "Collection.fetch(): Missing required argument 'ids'")); + return env.Undefined(); + } + if (obj.Has("outputFields") && !obj.Get("outputFields").IsUndefined()) { + if (!obj.Get("outputFields").IsArray()) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "Collection.fetch(): Expected a string or array " - "of strings for 'ids'")); + "Collection.fetch(): argument 'outputFields' must " + "be an array of strings")); return env.Undefined(); } + std::vector fields{}; + auto array = obj.Get("outputFields").As(); + fields.reserve(array.Length()); + for (uint32_t i = 0; i < array.Length(); i++) { + auto item = array.Get(i); + if (!item.IsString()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected an array of " + "strings for 'outputFields'")); + return env.Undefined(); + } + fields.emplace_back(item.As().Utf8Value()); + } + output_fields = std::move(fields); } - } else { - if (ids.IsString()) { - pks.emplace_back(ids.As().Utf8Value()); - } else { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "Collection.fetch(): Expected a string or array of " - "strings for 'ids'")); - return env.Undefined(); + if (obj.Has("includeVector") && !obj.Get("includeVector").IsUndefined()) { + if (!obj.Get("includeVector").IsBoolean()) { + ThrowIfNotOk( + env, + zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a boolean for 'includeVector'")); + return env.Undefined(); + } + include_vector = obj.Get("includeVector").As().Value(); } + + if (!parse_ids(obj.Get("ids"))) return env.Undefined(); } res = collection_->Fetch(pks, output_fields, include_vector); diff --git a/src/index.d.ts b/src/index.d.ts index c770e18..b4212f1 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -1124,6 +1124,13 @@ export declare class ZVecCollection { */ query(params: ZVecQuery): Promise; + /** + * Fetches documents by their IDs. + * @param ids - A single document ID or an array of document IDs to fetch. + * @returns An object mapping the requested IDs to their corresponding documents. If an ID is not found, it will not be present in the returned object. + */ + fetchSync(ids: string | string[]): Record; + /** * Fetches documents by their IDs. * @param params - Fetch parameters. From 4d109683ef29be5764a70c4407866031f55fcaf2 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 17:52:38 +0800 Subject: [PATCH 08/12] update tests --- tests/data/operations.test.ts | 12 ++++++++ tests/schema.test.ts | 57 ++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/tests/data/operations.test.ts b/tests/data/operations.test.ts index c2ab116..144da1c 100644 --- a/tests/data/operations.test.ts +++ b/tests/data/operations.test.ts @@ -48,6 +48,18 @@ describe('Data Operations Pipeline', () => { }); expectDoc(results[0], 42, 1, 1); }); + + it('should respect fetch projection controls', () => { + const doc = makeDoc(42, 1, 1); + const fetched = collection.fetchSync({ + ids: doc.id, + outputFields: ['title'], + includeVector: false + })[doc.id]; + expect(fetched.fields.title).toBe(doc.fields!.title); + expect(fetched.fields.price).toBeUndefined(); + expect(Object.keys(fetched.vectors)).toHaveLength(0); + }); }); diff --git a/tests/schema.test.ts b/tests/schema.test.ts index c5a143e..493d178 100644 --- a/tests/schema.test.ts +++ b/tests/schema.test.ts @@ -1,6 +1,7 @@ import { ZVecCollectionSchema, ZVecDataType, + ZVecDiskAnnIndexParams, ZVecHnswIndexParams, ZVecHnswRabitqIndexParams, ZVecIndexType, @@ -251,4 +252,58 @@ describe('CollectionSchema', () => { expect((vectors[1].indexParams as ZVecHnswRabitqIndexParams).efConstruction).toBe(500); expect((vectors[1].indexParams as ZVecHnswRabitqIndexParams).sampleCount).toBe(0); }); -}); \ No newline at end of file + + + (isLinuxX64 ? it : it.skip)('should parse DiskANN index params correctly', () => { + const schema = new ZVecCollectionSchema({ + name: 'test_diskann', + vectors: [ + { + name: 'vector1', + dataType: ZVecDataType.VECTOR_FP32, + dimension: 128, + indexParams: { + indexType: ZVecIndexType.DISKANN, + metricType: ZVecMetricType.L2, + maxDegree: 64, + listSize: 80, + pqChunkNum: 8, + quantizeType: ZVecQuantizeType.FP16 + } + }, + { + name: 'vector2', + dataType: ZVecDataType.VECTOR_FP32, + dimension: 64, + indexParams: { + indexType: ZVecIndexType.DISKANN + } + } + ] + }); + + expect(schema).toBeInstanceOf(ZVecCollectionSchema); + expect(schema.name).toBe('test_diskann'); + + const vectors = schema.vectors(); + expect(vectors.length).toBe(2); + expect(vectors[0].name).toBe('vector1'); + expect(vectors[0].dataType).toBe(ZVecDataType.VECTOR_FP32); + expect(vectors[0].dimension).toBe(128); + expect(vectors[0].indexParams!.indexType).toBe(ZVecIndexType.DISKANN); + expect(vectors[0].indexParams!.metricType).toBe(ZVecMetricType.L2); + expect((vectors[0].indexParams as ZVecDiskAnnIndexParams).maxDegree).toBe(64); + expect((vectors[0].indexParams as ZVecDiskAnnIndexParams).listSize).toBe(80); + expect((vectors[0].indexParams as ZVecDiskAnnIndexParams).pqChunkNum).toBe(8); + expect((vectors[0].indexParams as ZVecDiskAnnIndexParams).quantizeType).toBe(ZVecQuantizeType.FP16); + expect(vectors[1].name).toBe('vector2'); + expect(vectors[1].dataType).toBe(ZVecDataType.VECTOR_FP32); + expect(vectors[1].dimension).toBe(64); + expect(vectors[1].indexParams!.indexType).toBe(ZVecIndexType.DISKANN); + expect(vectors[1].indexParams!.metricType).toBe(ZVecMetricType.IP); + expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).maxDegree).toBe(100); + expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).listSize).toBe(50); + expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).pqChunkNum).toBe(0); + expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).quantizeType).toBe(ZVecQuantizeType.UNDEFINED); + }); +}); From 5e754a3ebc6ca06478ea74142d2087f0fc6dabcf Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 17:54:08 +0800 Subject: [PATCH 09/12] update --- src/zvec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zvec b/src/zvec index 0923f7c..b224b17 160000 --- a/src/zvec +++ b/src/zvec @@ -1 +1 @@ -Subproject commit 0923f7c6910d46907420e08f6225c82395dab9d3 +Subproject commit b224b17653758223f48ebf4ae3deb3217caf925a From 9f2e022131c0b947ccaef2b996fd976dd12cc476 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 19:53:44 +0800 Subject: [PATCH 10/12] fix libaio --- .github/workflows/publish.yml | 1 + .github/workflows/test-source.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2d84358..328c4e5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,6 +39,7 @@ jobs: - name: Install Dependencies run: | + dnf install -y libaio-devel curl -sL ${{ matrix.cmake_url }} | tar xz -C /usr/local --strip-components=1 npm install --ignore-scripts diff --git a/.github/workflows/test-source.yml b/.github/workflows/test-source.yml index 34b6380..beaefb5 100644 --- a/.github/workflows/test-source.yml +++ b/.github/workflows/test-source.yml @@ -35,6 +35,7 @@ jobs: - name: Install Dependencies run: | + dnf install -y libaio-devel curl -sL https://cmake.org/files/v3.27/cmake-3.27.9-linux-x86_64.tar.gz | tar xz -C /usr/local --strip-components=1 npm install --ignore-scripts From b379feb91b09050817176e53e552345688073ef6 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 22:04:25 +0800 Subject: [PATCH 11/12] update ut --- tests/data/helpers.ts | 20 ++++++++++-- tests/data/operations.test.ts | 59 ++++++++++++++++++++++++++++++++++- tests/schema.test.ts | 30 ++++++++++++++++++ 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/tests/data/helpers.ts b/tests/data/helpers.ts index 222ccda..011bf02 100644 --- a/tests/data/helpers.ts +++ b/tests/data/helpers.ts @@ -44,6 +44,11 @@ const TEST_SCHEMA = { dataType: ZVecDataType.FLOAT, nullable: true, indexParams: { indexType: ZVecIndexType.INVERT, enableRangeOptimization: true } + }, + { + name: 'content', + dataType: ZVecDataType.STRING, + indexParams: { indexType: ZVecIndexType.FTS } } ] }; @@ -57,6 +62,9 @@ export function createTestSchema(name: string): ZVecCollectionSchema { function title(k: number, version: number): string { return `Product_${k}_v${version}`; } function price(k: number, version: number): number { return (k + 0.99) * version; } +function content(k: number, version: number): string { + return `product ${k} version ${version} search content`; +} function dense(k: number, version: number): number[] { const s = k * 1000 + version; return Array.from({ length: DIMENSION }, (_, i) => Math.sin(s * (i + 1))); @@ -70,14 +78,22 @@ export function makeDoc(k: number, fieldVersion: number, vectorVersion: number): return { id: `doc_${k}`, vectors: { dense: dense(k, vectorVersion), sparse: sparse(k, vectorVersion) }, - fields: { title: title(k, fieldVersion), price: price(k, fieldVersion) } + fields: { + title: title(k, fieldVersion), + price: price(k, fieldVersion), + content: content(k, fieldVersion) + } }; } export function makeUpdate(k: number, fieldVersion: number): ZVecDocInput { return { id: `doc_${k}`, - fields: { title: title(k, fieldVersion), price: price(k, fieldVersion) } + fields: { + title: title(k, fieldVersion), + price: price(k, fieldVersion), + content: content(k, fieldVersion) + } }; } diff --git a/tests/data/operations.test.ts b/tests/data/operations.test.ts index 144da1c..bdb4518 100644 --- a/tests/data/operations.test.ts +++ b/tests/data/operations.test.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import { ZVecCollection, ZVecCreateAndOpen, + ZVecIndexType, ZVecInitialize, ZVecLogLevel, ZVecLogType, @@ -49,8 +50,11 @@ describe('Data Operations Pipeline', () => { expectDoc(results[0], 42, 1, 1); }); - it('should respect fetch projection controls', () => { + it('should fetch docs correctly', () => { const doc = makeDoc(42, 1, 1); + expectDoc(collection.fetchSync(doc.id)[doc.id], 42, 1, 1); + expectDoc(collection.fetchSync({ ids: doc.id })[doc.id], 42, 1, 1); + const fetched = collection.fetchSync({ ids: doc.id, outputFields: ['title'], @@ -59,6 +63,20 @@ describe('Data Operations Pipeline', () => { expect(fetched.fields.title).toBe(doc.fields!.title); expect(fetched.fields.price).toBeUndefined(); expect(Object.keys(fetched.vectors)).toHaveLength(0); + + const docs = [makeDoc(43, 1, 1), makeDoc(44, 1, 1)]; + const ids = docs.map(doc => doc.id); + const batchFetched = collection.fetchSync({ + ids, + outputFields: [] + }); + expect(new Set(Object.keys(batchFetched))).toEqual(new Set(ids)); + for (const doc of docs) { + expect(batchFetched[doc.id].id).toBe(doc.id); + expect(Object.keys(batchFetched[doc.id].fields)).toHaveLength(0); + expect(batchFetched[doc.id].vectors.dense).toBeDefined(); + expect(batchFetched[doc.id].vectors.sparse).toBeDefined(); + } }); }); @@ -144,6 +162,20 @@ describe('Data Operations Pipeline', () => { filter: `title = "Product_${k}_v1"` }).then(results => ({ k, results })); }); + const ftsQueries = targets.map(k => { + const doc = makeDoc(k, 1, 1); + return collection.query({ + fieldName: 'content', + fts: { matchString: doc.fields!.content }, + params: { + indexType: ZVecIndexType.FTS, + defaultOperator: 'AND' + }, + topk: 10, + outputFields: ['title'], + includeVector: false + }).then(results => ({ k, results })); + }); const denseResults = await Promise.all(denseQueries); for (const { k, results } of denseResults) { @@ -155,6 +187,14 @@ describe('Data Operations Pipeline', () => { expect(results.length).toBe(1); expectDoc(results[0], k, 1, 1); } + const ftsResults = await Promise.all(ftsQueries); + for (const { k, results } of ftsResults) { + expect(results.length).toBe(1); + expect(results[0].id).toBe(`doc_${k}`); + expect(results[0].fields.title).toBe(`Product_${k}_v1`); + expect(results[0].fields.content).toBeUndefined(); + expect(Object.keys(results[0].vectors)).toHaveLength(0); + } }); }); @@ -287,5 +327,22 @@ describe('Data Operations Pipeline', () => { expect(isZVecError(e)).toBe(true); } }); + + it('should throw clear fetch errors for invalid fetch options', () => { + const invalidCalls = [ + () => collection.fetchSync({} as any), + () => collection.fetchSync({ ids: 42 } as any), + () => collection.fetchSync({ ids: 'doc_1', outputFields: 'title' } as any), + () => collection.fetchSync({ ids: 'doc_1', includeVector: 'false' } as any) + ]; + for (const call of invalidCalls) { + try { + call(); + fail('expected to throw'); + } catch (e) { + expect(isZVecError(e)).toBe(true); + } + } + }); }); }); diff --git a/tests/schema.test.ts b/tests/schema.test.ts index 493d178..2f047ca 100644 --- a/tests/schema.test.ts +++ b/tests/schema.test.ts @@ -2,6 +2,7 @@ import { ZVecCollectionSchema, ZVecDataType, ZVecDiskAnnIndexParams, + ZVecFtsIndexParams, ZVecHnswIndexParams, ZVecHnswRabitqIndexParams, ZVecIndexType, @@ -306,4 +307,33 @@ describe('CollectionSchema', () => { expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).pqChunkNum).toBe(0); expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).quantizeType).toBe(ZVecQuantizeType.UNDEFINED); }); + + + it('should parse FTS index params correctly', () => { + const schema = new ZVecCollectionSchema({ + name: 'test_fts', + fields: { + name: 'content', + dataType: ZVecDataType.STRING, + indexParams: { + indexType: ZVecIndexType.FTS, + tokenizerName: 'whitespace', + filters: ['lowercase'], + extraParams: '{"jieba_dict_dir": "/path/to/jieba/dict"}' + } + } + }); + + expect(schema).toBeInstanceOf(ZVecCollectionSchema); + expect(schema.name).toBe('test_fts'); + + const fields = schema.fields(); + expect(fields.length).toBe(1); + expect(fields[0].name).toBe('content'); + expect(fields[0].dataType).toBe(ZVecDataType.STRING); + expect(fields[0].indexParams!.indexType).toBe(ZVecIndexType.FTS); + expect((fields[0].indexParams as ZVecFtsIndexParams).tokenizerName).toBe('whitespace'); + expect((fields[0].indexParams as ZVecFtsIndexParams).filters).toEqual(['lowercase']); + expect((fields[0].indexParams as ZVecFtsIndexParams).extraParams).toBe('{"jieba_dict_dir": "/path/to/jieba/dict"}'); + }); }); From 1768aa2513a1e5c8832e7200263046da994f0109 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Tue, 9 Jun 2026 22:08:59 +0800 Subject: [PATCH 12/12] update --- src/zvec | 2 +- tests/data/operations.test.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/zvec b/src/zvec index b224b17..da39a33 160000 --- a/src/zvec +++ b/src/zvec @@ -1 +1 @@ -Subproject commit b224b17653758223f48ebf4ae3deb3217caf925a +Subproject commit da39a33feb3d70a2ffc5ea7e59a1319954e18757 diff --git a/tests/data/operations.test.ts b/tests/data/operations.test.ts index bdb4518..2cb8df5 100644 --- a/tests/data/operations.test.ts +++ b/tests/data/operations.test.ts @@ -189,9 +189,10 @@ describe('Data Operations Pipeline', () => { } const ftsResults = await Promise.all(ftsQueries); for (const { k, results } of ftsResults) { + const doc = makeDoc(k, 1, 1); expect(results.length).toBe(1); - expect(results[0].id).toBe(`doc_${k}`); - expect(results[0].fields.title).toBe(`Product_${k}_v1`); + expect(results[0].id).toBe(doc.id); + expect(results[0].fields.title).toBe(doc.fields!.title); expect(results[0].fields.content).toBeUndefined(); expect(Object.keys(results[0].vectors)).toHaveLength(0); }