diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2d84358..328c4e5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,6 +39,7 @@ jobs: - name: Install Dependencies run: | + dnf install -y libaio-devel curl -sL ${{ matrix.cmake_url }} | tar xz -C /usr/local --strip-components=1 npm install --ignore-scripts diff --git a/.github/workflows/test-source.yml b/.github/workflows/test-source.yml index 34b6380..beaefb5 100644 --- a/.github/workflows/test-source.yml +++ b/.github/workflows/test-source.yml @@ -35,6 +35,7 @@ jobs: - name: Install Dependencies run: | + dnf install -y libaio-devel curl -sL https://cmake.org/files/v3.27/cmake-3.27.9-linux-x86_64.tar.gz | tar xz -C /usr/local --strip-components=1 npm install --ignore-scripts diff --git a/CMakeLists.txt b/CMakeLists.txt index b9cc5a0..87b7d5e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,10 @@ set(ZVEC_BUILD_TARGETS --target zvec_turbo --target zvec_db ) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64") + list(APPEND ZVEC_BUILD_TARGETS --target core_knn_diskann) + list(APPEND ZVEC_BYPRODUCTS ${ZVEC_LIB_DIR}/libzvec_diskann_plugin.so) +endif() set(ZVEC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release) if(MSVC) list(APPEND ZVEC_CMAKE_ARGS @@ -141,6 +145,7 @@ if (NOT WIN32) ${GLOG_LIB} ${GFLAGS_LIB} ${PROTOBUF_LIB} + FastPFOR lz4 ) else () @@ -165,6 +170,7 @@ else () ${GLOG_LIB} ${GFLAGS_LIB} ${PROTOBUF_LIB} + FastPFOR lz4 rpcrt4 shlwapi @@ -318,3 +324,12 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release") ) endif() endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64") + add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${ZVEC_LIB_DIR}/libzvec_diskann_plugin.so" + "$/libzvec_diskann_plugin.so" + COMMENT "Copying DiskANN runtime plugin" + ) +endif() diff --git a/package.json b/package.json index ba9be48..09b3451 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "src/index.d.mts", "scripts/install.js", "zvec_node_binding.node", + "libzvec_diskann_plugin.so", "README.md", "LICENSE" ], @@ -79,4 +80,4 @@ "dependencies": { "bindings": "^1.5.0" } -} +} \ No newline at end of file diff --git a/packages/bindings-linux-x64/package.json b/packages/bindings-linux-x64/package.json index 0bbe667..393f869 100644 --- a/packages/bindings-linux-x64/package.json +++ b/packages/bindings-linux-x64/package.json @@ -10,6 +10,7 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node" + "zvec_node_binding.node", + "libzvec_diskann_plugin.so" ] -} +} \ No newline at end of file diff --git a/scripts/build.js b/scripts/build.js index f9e120d..96590bf 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -56,8 +56,18 @@ try { const targetPath = path.join(platformPackageDir, 'zvec_node_binding.node'); fs.copyFileSync(binaryPath, targetPath); + // DiskANN currently ships as a runtime-loaded plugin. Keep it next to the + // addon binary so the C++ engine can auto-load it when DiskANN is used. + const diskAnnPluginPath = path.join(BUILD_TARGET_DIR, 'libzvec_diskann_plugin.so'); + if (fs.existsSync(diskAnnPluginPath)) { + fs.copyFileSync( + diskAnnPluginPath, + path.join(platformPackageDir, 'libzvec_diskann_plugin.so') + ); + } + console.log(`✅ Binary compiled and packaged for ${platformPackageName} at: ${targetPath}`); } catch (error) { console.error('❌ Error during build and packaging:', error.message); process.exit(1); -} \ No newline at end of file +} diff --git a/scripts/pack-local.js b/scripts/pack-local.js index aed3201..5c30bcb 100644 --- a/scripts/pack-local.js +++ b/scripts/pack-local.js @@ -26,6 +26,15 @@ try { fs.copyFileSync(targetPath, destPath); console.log(`Binary copied from ${targetPath} to ${destPath}`); + // DiskANN currently ships as a runtime-loaded plugin. Local packs bundle it + // beside the addon binary so the C++ engine can auto-load it when needed. + const pluginPath = path.join(platformPackageDir, 'libzvec_diskann_plugin.so'); + if (fs.existsSync(pluginPath)) { + const pluginDestPath = path.join(PACKAGE_ROOT, 'libzvec_diskann_plugin.so'); + fs.copyFileSync(pluginPath, pluginDestPath); + console.log(`DiskANN plugin copied from ${pluginPath} to ${pluginDestPath}`); + } + // Temporarily remove optionalDependencies from package.json // (local pack bundles the binary directly, no need for platform packages) const packageJsonPath = path.join(PACKAGE_ROOT, 'package.json'); @@ -44,4 +53,4 @@ try { } catch (error) { console.error('❌ Error during build and packaging:', error.message); process.exit(1); -} \ No newline at end of file +} diff --git a/src/binding/addon.cc b/src/binding/addon.cc index 2d4569c..d08c0e4 100644 --- a/src/binding/addon.cc +++ b/src/binding/addon.cc @@ -39,4 +39,4 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { } -NODE_API_MODULE(zvec_node_binding, Init) \ No newline at end of file +NODE_API_MODULE(zvec_node_binding, Init) diff --git a/src/binding/async_workers.cc b/src/binding/async_workers.cc index 4e0f9d1..7ad65e8 100644 --- a/src/binding/async_workers.cc +++ b/src/binding/async_workers.cc @@ -31,7 +31,7 @@ void DeleteByFilterWorker::OnError(const Napi::Error &error) { QueryWorker::QueryWorker(Napi::Env env, zvec::Collection::Ptr collection, zvec::CollectionSchema::Ptr schema, - zvec::VectorQuery query, + zvec::SearchQuery query, Napi::Promise::Deferred deferred) : Napi::AsyncWorker(env), collection_(collection), diff --git a/src/binding/async_workers.h b/src/binding/async_workers.h index 71ad3ad..316579d 100644 --- a/src/binding/async_workers.h +++ b/src/binding/async_workers.h @@ -3,8 +3,8 @@ #include #include -#include "zvec/db/collection.h" -#include "zvec/db/status.h" +#include +#include namespace binding { @@ -30,7 +30,7 @@ class DeleteByFilterWorker : public Napi::AsyncWorker { class QueryWorker : public Napi::AsyncWorker { public: QueryWorker(Napi::Env env, zvec::Collection::Ptr collection, - zvec::CollectionSchema::Ptr schema, zvec::VectorQuery query, + zvec::CollectionSchema::Ptr schema, zvec::SearchQuery query, Napi::Promise::Deferred deferred); void Execute() override; @@ -40,7 +40,7 @@ class QueryWorker : public Napi::AsyncWorker { private: zvec::Collection::Ptr collection_; zvec::CollectionSchema::Ptr schema_; - zvec::VectorQuery query_; + zvec::SearchQuery query_; Napi::Promise::Deferred deferred_; zvec::Status status_; zvec::DocPtrList results_; diff --git a/src/binding/collection.cc b/src/binding/collection.cc index ebef24a..1284f12 100644 --- a/src/binding/collection.cc +++ b/src/binding/collection.cc @@ -1,4 +1,5 @@ #include "collection.h" +#include #include "async_workers.h" #include "doc.h" #include "params.h" @@ -570,7 +571,7 @@ Napi::Value Collection::Query(const Napi::CallbackInfo &info) { return env.Undefined(); } - if (auto parsed_query = ParseVectorQuery(info[0], get_wrapped_schema()); + if (auto parsed_query = ParseSearchQuery(info[0], get_wrapped_schema()); parsed_query) { auto res = collection_->Query(parsed_query.value()); if (res) { @@ -604,7 +605,7 @@ Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { return deferred.Promise(); } - if (auto parsed_query = ParseVectorQuery(info[0], get_wrapped_schema()); + if (auto parsed_query = ParseSearchQuery(info[0], get_wrapped_schema()); parsed_query) { auto *worker = new QueryWorker(env, collection_, get_wrapped_schema(), std::move(parsed_query.value()), deferred); @@ -620,42 +621,96 @@ Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) { Napi::Value Collection::Fetch(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); if (ThrowIfClosed(env)) return env.Undefined(); - if (info.Length() != 1) { - ThrowIfNotOk( - env, zvec::Status::InvalidArgument( - "Collection.fetch(): Expected exactly 1 argument. " - "Argument must be an id string or an array of id strings.")); + if (info.Length() != 1 || + !(info[0].IsObject() || info[0].IsString() || info[0].IsArray())) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected exactly 1 argument: " + "(ids: string | string[]) or " + "({ ids: string | string[], outputFields?: string[], " + "includeVector?: boolean }).")); return env.Undefined(); } - std::vector pks{}; zvec::Result res; - - if (info[0].IsArray()) { - Napi::Array pkArray = info[0].As(); - uint32_t length = pkArray.Length(); - pks.reserve(length); - for (uint32_t i = 0; i < length; i++) { - if (pkArray.Get(i).IsString()) { - pks.emplace_back(pkArray.Get(i).As().Utf8Value()); - } else { - ThrowIfNotOk(env, - zvec::Status::InvalidArgument("Expected a string for id")); - return env.Undefined(); + std::vector pks{}; + std::optional> output_fields{}; + bool include_vector{true}; + + auto parse_ids = [&](const Napi::Value &ids) -> bool { + if (ids.IsArray()) { + Napi::Array pkArray = ids.As(); + uint32_t length = pkArray.Length(); + pks.reserve(length); + for (uint32_t i = 0; i < length; i++) { + if (pkArray.Get(i).IsString()) { + pks.emplace_back(pkArray.Get(i).As().Utf8Value()); + } else { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a string or array " + "of strings for 'ids'")); + return false; + } } + return true; + } + if (ids.IsString()) { + pks.emplace_back(ids.As().Utf8Value()); + return true; } - res = collection_->Fetch(pks); + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a string or array of " + "strings for 'ids'")); + return false; + }; + + if (info[0].IsString() || info[0].IsArray()) { + if (!parse_ids(info[0])) return env.Undefined(); } else { - if (info[0].IsString()) { - pks.emplace_back(info[0].As().Utf8Value()); - } else { + auto obj = info[0].As(); + if (!obj.Has("ids")) { ThrowIfNotOk(env, - zvec::Status::InvalidArgument("Expected a string for id")); + zvec::Status::InvalidArgument( + "Collection.fetch(): Missing required argument 'ids'")); return env.Undefined(); } - res = collection_->Fetch(pks); + if (obj.Has("outputFields") && !obj.Get("outputFields").IsUndefined()) { + if (!obj.Get("outputFields").IsArray()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): argument 'outputFields' must " + "be an array of strings")); + return env.Undefined(); + } + std::vector fields{}; + auto array = obj.Get("outputFields").As(); + fields.reserve(array.Length()); + for (uint32_t i = 0; i < array.Length(); i++) { + auto item = array.Get(i); + if (!item.IsString()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Collection.fetch(): Expected an array of " + "strings for 'outputFields'")); + return env.Undefined(); + } + fields.emplace_back(item.As().Utf8Value()); + } + output_fields = std::move(fields); + } + if (obj.Has("includeVector") && !obj.Get("includeVector").IsUndefined()) { + if (!obj.Get("includeVector").IsBoolean()) { + ThrowIfNotOk( + env, + zvec::Status::InvalidArgument( + "Collection.fetch(): Expected a boolean for 'includeVector'")); + return env.Undefined(); + } + include_vector = obj.Get("includeVector").As().Value(); + } + + if (!parse_ids(obj.Get("ids"))) return env.Undefined(); } + res = collection_->Fetch(pks, output_fields, include_vector); + if (res) { Napi::Object obj = Napi::Object::New(env); const zvec::DocPtrMap &map = res.value(); @@ -1006,4 +1061,4 @@ Napi::Value Collection::DropIndex(const Napi::CallbackInfo &info) { } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/collection.h b/src/binding/collection.h index 1f48f6a..e317e52 100644 --- a/src/binding/collection.h +++ b/src/binding/collection.h @@ -2,7 +2,7 @@ #include -#include "zvec/db/collection.h" +#include #include "addon.h" @@ -85,4 +85,4 @@ class Collection : public Napi::ObjectWrap { }; -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/config.cc b/src/binding/config.cc index d92ed4c..9586f9a 100644 --- a/src/binding/config.cc +++ b/src/binding/config.cc @@ -1,5 +1,5 @@ #include "config.h" -#include "zvec/db/config.h" +#include #include "types.h" @@ -17,11 +17,11 @@ Napi::Object CreateLogTypeObject(Napi::Env env) { Napi::Object CreateLogLevelObject(Napi::Env env) { Napi::Object obj = Napi::Object::New(env); - obj.Set("DEBUG", static_cast(zvec::GlobalConfig::LogLevel::DEBUG)); - obj.Set("INFO", static_cast(zvec::GlobalConfig::LogLevel::INFO)); - obj.Set("WARN", static_cast(zvec::GlobalConfig::LogLevel::WARN)); - obj.Set("ERROR", static_cast(zvec::GlobalConfig::LogLevel::ERROR)); - obj.Set("FATAL", static_cast(zvec::GlobalConfig::LogLevel::FATAL)); + obj.Set("DEBUG", static_cast(zvec::GlobalConfig::LogLevel::kDebug)); + obj.Set("INFO", static_cast(zvec::GlobalConfig::LogLevel::kInfo)); + obj.Set("WARN", static_cast(zvec::GlobalConfig::LogLevel::kWarn)); + obj.Set("ERROR", static_cast(zvec::GlobalConfig::LogLevel::kError)); + obj.Set("FATAL", static_cast(zvec::GlobalConfig::LogLevel::kFatal)); obj.Freeze(); return obj; } @@ -41,7 +41,7 @@ Napi::Value Initialize(const Napi::CallbackInfo &info) { if (obj.Has("logType") || obj.Has("logLevel")) { std::string log_type = "console"; - zvec::GlobalConfig::LogLevel log_level{zvec::GlobalConfig::LogLevel::WARN}; + zvec::GlobalConfig::LogLevel log_level{zvec::GlobalConfig::LogLevel::kWarn}; if (obj.Has("logType")) { if (obj.Get("logType").IsString()) { log_type = obj.Get("logType").As().Utf8Value(); @@ -160,6 +160,29 @@ Napi::Value Initialize(const Napi::CallbackInfo &info) { } } + if (obj.Has("ftsBruteForceByKeysRatio")) { + if (obj.Get("ftsBruteForceByKeysRatio").IsNumber()) { + config.fts_brute_force_by_keys_ratio = + obj.Get("ftsBruteForceByKeysRatio").As().FloatValue(); + } else { + ThrowIfNotOk(env, + zvec::Status::InvalidArgument("Expected a number for " + "'ftsBruteForceByKeysRatio'")); + return env.Undefined(); + } + } + + if (obj.Has("jiebaDictDir")) { + if (obj.Get("jiebaDictDir").IsString()) { + config.jieba_dict_dir = + obj.Get("jiebaDictDir").As().Utf8Value(); + } else { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "Expected a string for 'jiebaDictDir'")); + return env.Undefined(); + } + } + if (auto s = zvec::GlobalConfig::Instance().Initialize(config); !s.ok()) { ThrowIfNotOk(env, s); } @@ -167,12 +190,36 @@ Napi::Value Initialize(const Napi::CallbackInfo &info) { } +Napi::Value SetDefaultJiebaDictDir(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() != 1 || !info[0].IsString()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "setDefaultJiebaDictDir() expects exactly one " + "argument: (dir: string)")); + return env.Undefined(); + } + zvec::GlobalConfig::Instance().set_default_jieba_dict_dir( + info[0].As().Utf8Value()); + return env.Undefined(); +} + + +Napi::Value GetDefaultJiebaDictDir(const Napi::CallbackInfo &info) { + return Napi::String::New(info.Env(), + zvec::GlobalConfig::Instance().jieba_dict_dir()); +} + + Napi::Object InitConfig(Napi::Env env, Napi::Object exports) { exports.Set("LogType", CreateLogTypeObject(env)); exports.Set("LogLevel", CreateLogLevelObject(env)); exports.Set("initialize", Napi::Function::New(env, Initialize)); + exports.Set("setDefaultJiebaDictDir", + Napi::Function::New(env, SetDefaultJiebaDictDir)); + exports.Set("getDefaultJiebaDictDir", + Napi::Function::New(env, GetDefaultJiebaDictDir)); return exports; } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/doc.h b/src/binding/doc.h index c0de686..f090a18 100644 --- a/src/binding/doc.h +++ b/src/binding/doc.h @@ -2,7 +2,7 @@ #include -#include "zvec/db/doc.h" +#include namespace binding { @@ -63,4 +63,4 @@ Napi::Array CreateScalarArray(Napi::Env &env, const std::string &fieldName, zvec::FieldSchema *schema, zvec::Doc::Ptr &doc); -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/params.cc b/src/binding/params.cc index 1d1f640..1ff105c 100644 --- a/src/binding/params.cc +++ b/src/binding/params.cc @@ -29,12 +29,16 @@ zvec::Result ParseIndexParams( return ParseFlatIndexParams(obj); case zvec::IndexType::HNSW: return ParseHnswIndexParams(obj); - case zvec::IndexType::IVF: - return ParseIVFIndexParams(obj); case zvec::IndexType::HNSW_RABITQ: return ParseHnswRabitqIndexParams(obj); + case zvec::IndexType::IVF: + return ParseIVFIndexParams(obj); + case zvec::IndexType::DISKANN: + return ParseDiskAnnIndexParams(obj); case zvec::IndexType::INVERT: return ParseInvertIndexParams(obj); + case zvec::IndexType::FTS: + return ParseFtsIndexParams(obj); default: uint32_t type_val = static_cast(index_type); return tl::make_unexpected(zvec::Status::InvalidArgument( @@ -49,12 +53,16 @@ Napi::Object CreateIndexParams(Napi::Env env, zvec::IndexParams::Ptr params) { return CreateFlatIndexParams(env, params); case zvec::IndexType::HNSW: return CreateHnswIndexParams(env, params); - case zvec::IndexType::IVF: - return CreateIVFIndexParams(env, params); case zvec::IndexType::HNSW_RABITQ: return CreateHnswRabitqIndexParams(env, params); + case zvec::IndexType::IVF: + return CreateIVFIndexParams(env, params); + case zvec::IndexType::DISKANN: + return CreateDiskAnnIndexParams(env, params); case zvec::IndexType::INVERT: return CreateInvertIndexParams(env, params); + case zvec::IndexType::FTS: + return CreateFtsIndexParams(env, params); default: return Napi::Object::New(env); } @@ -140,8 +148,19 @@ zvec::Result ParseHnswIndexParams( } } + bool use_contiguous_memory{false}; + if (obj.Has("useContiguousMemory")) { + if (obj.Get("useContiguousMemory").IsBoolean()) { + use_contiguous_memory = + obj.Get("useContiguousMemory").As().Value(); + } else { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a boolean for 'useContiguousMemory'")); + } + } + return std::make_shared( - metric_type, m, ef_construction, quantize_type); + metric_type, m, ef_construction, quantize_type, use_contiguous_memory); } @@ -154,6 +173,90 @@ Napi::Object CreateHnswIndexParams(Napi::Env env, obj.Set("m", hnsw_params->m()); obj.Set("efConstruction", hnsw_params->ef_construction()); obj.Set("quantizeType", static_cast(hnsw_params->quantize_type())); + obj.Set("useContiguousMemory", hnsw_params->use_contiguous_memory()); + return obj; +} + + +zvec::Result ParseHnswRabitqIndexParams( + const Napi::Object &obj) { + zvec::MetricType metric_type{zvec::MetricType::IP}; + if (obj.Has("metricType")) { + auto parsed_metric_type = ParseMetricType(obj.Get("metricType")); + if (parsed_metric_type) { + metric_type = parsed_metric_type.value(); + } else { + return tl::make_unexpected(parsed_metric_type.error()); + } + } + + int total_bits{7}; + if (obj.Has("totalBits")) { + if (obj.Get("totalBits").IsNumber()) { + total_bits = obj.Get("totalBits").As(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'totalBits'")); + } + } + + int num_clusters{16}; + if (obj.Has("numClusters")) { + if (obj.Get("numClusters").IsNumber()) { + num_clusters = obj.Get("numClusters").As(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'numClusters'")); + } + } + + int m{50}; + if (obj.Has("m")) { + if (obj.Get("m").IsNumber()) { + m = obj.Get("m").As(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'm'")); + } + } + + int ef_construction{500}; + if (obj.Has("efConstruction")) { + if (obj.Get("efConstruction").IsNumber()) { + ef_construction = obj.Get("efConstruction").As(); + } else { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a number for 'efConstruction'")); + } + } + + int sample_count{0}; + if (obj.Has("sampleCount")) { + if (obj.Get("sampleCount").IsNumber()) { + sample_count = obj.Get("sampleCount").As(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'sampleCount'")); + } + } + + return std::make_shared( + metric_type, total_bits, num_clusters, m, ef_construction, sample_count); +} + + +Napi::Object CreateHnswRabitqIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params) { + auto obj = Napi::Object::New(env); + auto rabitq_params = + std::dynamic_pointer_cast(params); + obj.Set("indexType", static_cast(rabitq_params->type())); + obj.Set("metricType", static_cast(rabitq_params->metric_type())); + obj.Set("totalBits", rabitq_params->total_bits()); + obj.Set("numClusters", rabitq_params->num_clusters()); + obj.Set("m", rabitq_params->m()); + obj.Set("efConstruction", rabitq_params->ef_construction()); + obj.Set("sampleCount", rabitq_params->sample_count()); return obj; } @@ -220,7 +323,7 @@ Napi::Object CreateIVFIndexParams(Napi::Env env, } -zvec::Result ParseHnswRabitqIndexParams( +zvec::Result ParseDiskAnnIndexParams( const Napi::Object &obj) { zvec::MetricType metric_type{zvec::MetricType::IP}; if (obj.Has("metricType")) { @@ -232,73 +335,63 @@ zvec::Result ParseHnswRabitqIndexParams( } } - int total_bits{7}; - if (obj.Has("totalBits")) { - if (obj.Get("totalBits").IsNumber()) { - total_bits = obj.Get("totalBits").As(); + int max_degree{100}; + if (obj.Has("maxDegree")) { + if (obj.Get("maxDegree").IsNumber()) { + max_degree = obj.Get("maxDegree").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'totalBits'")); + zvec::Status::InvalidArgument("Expected a number for 'maxDegree'")); } } - int num_clusters{16}; - if (obj.Has("numClusters")) { - if (obj.Get("numClusters").IsNumber()) { - num_clusters = obj.Get("numClusters").As(); + int list_size{50}; + if (obj.Has("listSize")) { + if (obj.Get("listSize").IsNumber()) { + list_size = obj.Get("listSize").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'numClusters'")); + zvec::Status::InvalidArgument("Expected a number for 'listSize'")); } } - int m{50}; - if (obj.Has("m")) { - if (obj.Get("m").IsNumber()) { - m = obj.Get("m").As(); + int pq_chunk_num{0}; + if (obj.Has("pqChunkNum")) { + if (obj.Get("pqChunkNum").IsNumber()) { + pq_chunk_num = obj.Get("pqChunkNum").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'm'")); + zvec::Status::InvalidArgument("Expected a number for 'pqChunkNum'")); } } - int ef_construction{500}; - if (obj.Has("efConstruction")) { - if (obj.Get("efConstruction").IsNumber()) { - ef_construction = obj.Get("efConstruction").As(); - } else { - return tl::make_unexpected(zvec::Status::InvalidArgument( - "Expected a number for 'efConstruction'")); - } - } - - int sample_count{0}; - if (obj.Has("sampleCount")) { - if (obj.Get("sampleCount").IsNumber()) { - sample_count = obj.Get("sampleCount").As(); + zvec::QuantizeType quantize_type{zvec::QuantizeType::UNDEFINED}; + if (obj.Has("quantizeType")) { + auto parsed_quantize_type = ParseQuantizeType(obj.Get("quantizeType")); + if (parsed_quantize_type) { + quantize_type = parsed_quantize_type.value(); } else { - return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'sampleCount'")); + return tl::make_unexpected(parsed_quantize_type.error()); } } - return std::make_shared( - metric_type, total_bits, num_clusters, m, ef_construction, sample_count); + return std::make_shared( + metric_type, max_degree, list_size, pq_chunk_num, quantize_type); } -Napi::Object CreateHnswRabitqIndexParams(Napi::Env env, - zvec::IndexParams::Ptr params) { +Napi::Object CreateDiskAnnIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params) { auto obj = Napi::Object::New(env); - auto rabitq_params = - std::dynamic_pointer_cast(params); - obj.Set("indexType", static_cast(rabitq_params->type())); - obj.Set("metricType", static_cast(rabitq_params->metric_type())); - obj.Set("totalBits", rabitq_params->total_bits()); - obj.Set("numClusters", rabitq_params->num_clusters()); - obj.Set("m", rabitq_params->m()); - obj.Set("efConstruction", rabitq_params->ef_construction()); - obj.Set("sampleCount", rabitq_params->sample_count()); + auto diskann_params = + std::dynamic_pointer_cast(params); + obj.Set("indexType", static_cast(diskann_params->type())); + obj.Set("metricType", static_cast(diskann_params->metric_type())); + obj.Set("maxDegree", diskann_params->max_degree()); + obj.Set("listSize", diskann_params->list_size()); + obj.Set("pqChunkNum", diskann_params->pq_chunk_num()); + obj.Set("quantizeType", + static_cast(diskann_params->quantize_type())); return obj; } @@ -343,24 +436,88 @@ Napi::Object CreateInvertIndexParams(Napi::Env env, } -zvec::Result ParseVectorQuery( +zvec::Result> ParseFtsIndexParams( + const Napi::Object &obj) { + std::string tokenizer_name{"standard"}; + if (obj.Has("tokenizerName")) { + if (obj.Get("tokenizerName").IsString()) { + tokenizer_name = obj.Get("tokenizerName").As().Utf8Value(); + } else { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a string for 'tokenizerName'")); + } + } + + std::vector filters{"lowercase"}; + if (obj.Has("filters")) { + if (!obj.Get("filters").IsArray()) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected an array of strings for 'filters'")); + } + filters.clear(); + Napi::Array array = obj.Get("filters").As(); + for (uint32_t i = 0; i < array.Length(); i++) { + auto item = array.Get(i); + if (!item.IsString()) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected an array of strings for 'filters'")); + } + filters.emplace_back(item.As().Utf8Value()); + } + } + + std::string extra_params{}; + if (obj.Has("extraParams")) { + if (obj.Get("extraParams").IsString()) { + extra_params = obj.Get("extraParams").As().Utf8Value(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a string for 'extraParams'")); + } + } + + return std::make_shared(tokenizer_name, filters, + extra_params); +} + + +Napi::Object CreateFtsIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params) { + auto obj = Napi::Object::New(env); + auto fts_params = std::dynamic_pointer_cast(params); + obj.Set("indexType", static_cast(fts_params->type())); + obj.Set("tokenizerName", fts_params->tokenizer_name()); + auto filters = Napi::Array::New(env, fts_params->filters().size()); + for (size_t i = 0; i < fts_params->filters().size(); i++) { + filters.Set(i, fts_params->filters()[i]); + } + obj.Set("filters", filters); + obj.Set("extraParams", fts_params->extra_params()); + return obj; +} + + +zvec::Result ParseSearchQuery( const Napi::Value &value, zvec::CollectionSchema::Ptr schema) { if (!value.IsObject()) { return tl::make_unexpected( zvec::Status::InvalidArgument("Expected an object for Query")); } - zvec::VectorQuery query{}; + zvec::SearchQuery query{}; + query.topk_ = 10; auto obj = value.As(); zvec::FieldSchema *field_schema{nullptr}; if (obj.Has("fieldName")) { if (obj.Get("fieldName").IsString()) { - query.field_name_ = obj.Get("fieldName").As().Utf8Value(); - field_schema = schema->get_field(query.field_name_); + query.target_.field_name_ = + obj.Get("fieldName").As().Utf8Value(); + field_schema = schema->get_field(query.target_.field_name_); if (!field_schema) { - return tl::make_unexpected(zvec::Status::InvalidArgument( - "'", query.field_name_, "' not found in collection schema")); + return tl::make_unexpected( + zvec::Status::InvalidArgument("'", query.target_.field_name_, + "' not found in collection schema")); } } else { return tl::make_unexpected(zvec::Status::InvalidArgument( @@ -376,7 +533,19 @@ zvec::Result ParseVectorQuery( zvec::Status::InvalidArgument("Expected a number for 'topk'")); } } - if (obj.Has("vector")) { + + const bool has_vector = obj.Has("vector") && + !obj.Get("vector").IsUndefined() && + !obj.Get("vector").IsNull(); + const bool has_fts = obj.Has("fts") && !obj.Get("fts").IsUndefined() && + !obj.Get("fts").IsNull(); + + if (has_vector && has_fts) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Cannot combine 'vector' and 'fts' in a single Query")); + } + + if (has_vector) { if (!field_schema) { return tl::make_unexpected(zvec::Status::InvalidArgument( "A vector name must be provided when performing vector queries")); @@ -392,27 +561,32 @@ zvec::Result ParseVectorQuery( } } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Field '", query.field_name_, + zvec::Status::InvalidArgument("Field '", query.target_.field_name_, "' is not a vector field and cannot be " "queried using vector operations.")); } - if (obj.Has("params")) { - auto parsed_params = ParseQueryParams(obj.Get("params")); - if (parsed_params) { - if (parsed_params.value()->type() == field_schema->index_type()) { - query.query_params_ = parsed_params.value(); - } else { - return tl::make_unexpected(zvec::Status::InvalidArgument( - "Unmatched queryParams type in Query")); - } - } else { - return tl::make_unexpected(parsed_params.error()); - } - } - } else { - if (field_schema) { + } else if (has_fts) { + if (!field_schema) { return tl::make_unexpected(zvec::Status::InvalidArgument( - "A vector must be provided when performing vector queries")); + "A field name must be provided when performing FTS queries")); + } + auto parsed_fts = ParseFtsClause(obj.Get("fts")); + if (parsed_fts) { + query.target_.clause_ = std::move(parsed_fts.value()); + } else { + return tl::make_unexpected(parsed_fts.error()); + } + } else if (field_schema) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "A vector or FTS clause must be provided when 'fieldName' is set")); + } + + if (obj.Has("params")) { + auto parsed_params = ParseQueryParams(obj.Get("params")); + if (parsed_params) { + query.target_.query_params_ = parsed_params.value(); + } else { + return tl::make_unexpected(parsed_params.error()); } } if (obj.Has("filter")) { @@ -456,7 +630,7 @@ zvec::Result ParseVectorQuery( zvec::Status ParseVectorToString(const Napi::Value &value, zvec::FieldSchema *schema, - zvec::VectorQuery *query) { + zvec::SearchQuery *query) { if (value.IsTypedArray()) { auto array = value.As(); auto ta_type = array.TypedArrayType(); @@ -476,7 +650,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, std::memcpy(buf.data() + (i * sizeof(uint16_t)), &val, sizeof(uint16_t)); } - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } case zvec::DataType::VECTOR_FP32: { @@ -489,7 +663,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, std::string buf; buf.resize(ta_length * sizeof(float)); std::memcpy(buf.data(), float32Array.Data(), ta_length * sizeof(float)); - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } case zvec::DataType::VECTOR_INT8: { @@ -501,7 +675,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, std::string buf; buf.resize(ta_length * sizeof(int8_t)); std::memcpy(buf.data(), int8Array.Data(), ta_length * sizeof(int8_t)); - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } default: { @@ -528,7 +702,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, "' must be numbers"); } } - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } case zvec::DataType::VECTOR_FP32: { @@ -545,7 +719,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, "' must be numbers"); } } - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } case zvec::DataType::VECTOR_INT8: { @@ -563,7 +737,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, "' must be numbers"); } } - query->query_vector_ = buf; + query->target_.set_vector(std::move(buf)); break; } default: { @@ -582,7 +756,7 @@ zvec::Status ParseVectorToString(const Napi::Value &value, zvec::Status ParseVectorToMap(const Napi::Value &value, zvec::FieldSchema *schema, - zvec::VectorQuery *query) { + zvec::SearchQuery *query) { if (!value.IsObject()) { return zvec::Status::InvalidArgument("Expected sparse vector[", schema->name(), "] to be an object"); @@ -623,8 +797,7 @@ zvec::Status ParseVectorToMap(const Napi::Value &value, indices_ptr[i] = index; values_ptr[i] = vectorValue.As().FloatValue(); } - query->query_sparse_indices_ = std::move(indices); - query->query_sparse_values_ = std::move(values); + query->target_.set_sparse_vector(std::move(indices), std::move(values)); break; } case zvec::DataType::SPARSE_VECTOR_FP16: { @@ -658,8 +831,7 @@ zvec::Status ParseVectorToMap(const Napi::Value &value, zvec::ailego::Float16 val = vectorValue.As().FloatValue(); values_ptr[i] = val; } - query->query_sparse_indices_ = std::move(indices); - query->query_sparse_values_ = std::move(values); + query->target_.set_sparse_vector(std::move(indices), std::move(values)); break; } default: { @@ -671,6 +843,33 @@ zvec::Status ParseVectorToMap(const Napi::Value &value, } +zvec::Result ParseFtsClause(const Napi::Value &value) { + if (!value.IsObject()) { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected an object for 'fts'")); + } + auto obj = value.As(); + zvec::FtsClause fts_clause{}; + if (obj.Has("queryString")) { + if (!obj.Get("queryString").IsString()) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a string for 'queryString' in 'fts'")); + } + fts_clause.query_string_ = + obj.Get("queryString").As().Utf8Value(); + } + if (obj.Has("matchString")) { + if (!obj.Get("matchString").IsString()) { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a string for 'matchString' in 'fts'")); + } + fts_clause.match_string_ = + obj.Get("matchString").As().Utf8Value(); + } + return fts_clause; +} + + zvec::Result ParseQueryParams( const Napi::Value &value) { if (!value.IsObject()) { @@ -696,13 +895,17 @@ zvec::Result ParseQueryParams( "Flat index type is not allowed for QueryParams")); case zvec::IndexType::HNSW: return ParseHnswQueryParams(obj); - case zvec::IndexType::IVF: - return ParseIVFQueryParams(obj); case zvec::IndexType::HNSW_RABITQ: return ParseHnswRabitqQueryParams(obj); + case zvec::IndexType::IVF: + return ParseIVFQueryParams(obj); + case zvec::IndexType::DISKANN: + return ParseDiskAnnQueryParams(obj); case zvec::IndexType::INVERT: return tl::make_unexpected(zvec::Status::InvalidArgument( "Inverted index type is not allowed for QueryParams")); + case zvec::IndexType::FTS: + return ParseFtsQueryParams(obj); default: uint32_t type_val = static_cast(query_type); return tl::make_unexpected(zvec::Status::InvalidArgument( @@ -764,15 +967,15 @@ zvec::Result ParseHnswQueryParams( } -zvec::Result ParseIVFQueryParams( +zvec::Result ParseHnswRabitqQueryParams( const Napi::Object &obj) { - int nprobe{10}; - if (obj.Has("nprobe")) { - if (obj.Get("nprobe").IsNumber()) { - nprobe = obj.Get("nprobe").As().Int32Value(); + int ef{300}; + if (obj.Has("ef")) { + if (obj.Get("ef").IsNumber()) { + ef = obj.Get("ef").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'nprobe'")); + zvec::Status::InvalidArgument("Expected a number for 'ef'")); } } @@ -806,19 +1009,20 @@ zvec::Result ParseIVFQueryParams( } } - return std::make_shared(nprobe, is_using_refiner); + return std::make_shared(ef, radius, is_linear, + is_using_refiner); } -zvec::Result ParseHnswRabitqQueryParams( +zvec::Result ParseIVFQueryParams( const Napi::Object &obj) { - int ef{300}; - if (obj.Has("ef")) { - if (obj.Get("ef").IsNumber()) { - ef = obj.Get("ef").As().Int32Value(); + int nprobe{10}; + if (obj.Has("nprobe")) { + if (obj.Get("nprobe").IsNumber()) { + nprobe = obj.Get("nprobe").As().Int32Value(); } else { return tl::make_unexpected( - zvec::Status::InvalidArgument("Expected a number for 'ef'")); + zvec::Status::InvalidArgument("Expected a number for 'nprobe'")); } } @@ -852,8 +1056,42 @@ zvec::Result ParseHnswRabitqQueryParams( } } - return std::make_shared(ef, radius, is_linear, - is_using_refiner); + auto params = + std::make_shared(nprobe, is_using_refiner); + params->set_radius(radius); + params->set_is_linear(is_linear); + return params; +} + + +zvec::Result ParseDiskAnnQueryParams( + const Napi::Object &obj) { + int list_size{300}; + if (obj.Has("listSize")) { + if (obj.Get("listSize").IsNumber()) { + list_size = obj.Get("listSize").As().Int32Value(); + } else { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'listSize'")); + } + } + return std::make_shared(list_size); +} + + +zvec::Result ParseFtsQueryParams( + const Napi::Object &obj) { + auto params = std::make_shared(); + if (obj.Has("defaultOperator")) { + if (obj.Get("defaultOperator").IsString()) { + params->set_default_operator( + obj.Get("defaultOperator").As().Utf8Value()); + } else { + return tl::make_unexpected(zvec::Status::InvalidArgument( + "Expected a string for 'defaultOperator'")); + } + } + return params; } @@ -956,4 +1194,4 @@ zvec::Result ParseOptimizeOptions( } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/params.h b/src/binding/params.h index 8afea6f..338f3ea 100644 --- a/src/binding/params.h +++ b/src/binding/params.h @@ -2,10 +2,10 @@ #include -#include "zvec/db/doc.h" -#include "zvec/db/index_params.h" -#include "zvec/db/options.h" -#include "zvec/db/status.h" +#include +#include +#include +#include namespace binding { @@ -28,36 +28,49 @@ zvec::Result ParseHnswIndexParams( Napi::Object CreateHnswIndexParams(Napi::Env env, zvec::IndexParams::Ptr params); +zvec::Result ParseHnswRabitqIndexParams( + const Napi::Object &obj); + +Napi::Object CreateHnswRabitqIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params); + zvec::Result ParseIVFIndexParams( const Napi::Object &obj); Napi::Object CreateIVFIndexParams(Napi::Env env, zvec::IndexParams::Ptr params); -zvec::Result ParseHnswRabitqIndexParams( +zvec::Result ParseDiskAnnIndexParams( const Napi::Object &obj); -Napi::Object CreateHnswRabitqIndexParams(Napi::Env env, - zvec::IndexParams::Ptr params); +Napi::Object CreateDiskAnnIndexParams(Napi::Env env, + zvec::IndexParams::Ptr params); zvec::Result ParseInvertIndexParams( const Napi::Object &obj); Napi::Object CreateInvertIndexParams(Napi::Env env, zvec::IndexParams::Ptr params); + +zvec::Result> ParseFtsIndexParams( + const Napi::Object &obj); + +Napi::Object CreateFtsIndexParams(Napi::Env env, zvec::IndexParams::Ptr params); /*** Index Parameters ***/ /*** Query Parameters ***/ -zvec::Result ParseVectorQuery( +zvec::Result ParseSearchQuery( const Napi::Value &value, zvec::CollectionSchema::Ptr schema); zvec::Status ParseVectorToString(const Napi::Value &value, zvec::FieldSchema *schema, - zvec::VectorQuery *query); + zvec::SearchQuery *query); zvec::Status ParseVectorToMap(const Napi::Value &value, zvec::FieldSchema *schema, - zvec::VectorQuery *query); + zvec::SearchQuery *query); + +zvec::Result ParseFtsClause(const Napi::Value &value); zvec::Result ParseQueryParams(const Napi::Value &value); @@ -67,10 +80,16 @@ zvec::Result ParseFlatQueryParams( zvec::Result ParseHnswQueryParams( const Napi::Object &obj); +zvec::Result ParseHnswRabitqQueryParams( + const Napi::Object &obj); + zvec::Result ParseIVFQueryParams( const Napi::Object &obj); -zvec::Result ParseHnswRabitqQueryParams( +zvec::Result ParseDiskAnnQueryParams( + const Napi::Object &obj); + +zvec::Result ParseFtsQueryParams( const Napi::Object &obj); /*** Query Parameters ***/ @@ -96,4 +115,4 @@ zvec::Result ParseOptimizeOptions( /*** Collection-Level Options ***/ -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/schema.cc b/src/binding/schema.cc index fd14e2c..df656bb 100644 --- a/src/binding/schema.cc +++ b/src/binding/schema.cc @@ -182,8 +182,8 @@ CollectionSchema::CollectionSchema(const Napi::CallbackInfo &info) if (info.Length() != 1) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "CollectionSchema constructor expects 1 argument " - "(schema definition object): { name: string, fields: " - "[...], vectors: [...] }")); + "(schema definition object): { name: string, " + "fields?: [...], vectors?: [...] }")); return; } @@ -219,35 +219,31 @@ CollectionSchema::CollectionSchema(const Napi::CallbackInfo &info) std::vector fields; - if (!obj.Has("vectors")) { - ThrowIfNotOk( - env, zvec::Status::InvalidArgument( - "Missing required argument 'vectors' in CollectionSchema")); - return; - } - Napi::Value vectorsValue = obj.Get("vectors"); - if (vectorsValue.IsArray()) { - Napi::Array vectorsArray = vectorsValue.As(); - for (uint32_t i = 0; i < vectorsArray.Length(); i++) { - auto parsed_vector_schema = ParseVectorSchema(vectorsArray.Get(i)); + if (obj.Has("vectors")) { + Napi::Value vectorsValue = obj.Get("vectors"); + if (vectorsValue.IsArray()) { + Napi::Array vectorsArray = vectorsValue.As(); + for (uint32_t i = 0; i < vectorsArray.Length(); i++) { + auto parsed_vector_schema = ParseVectorSchema(vectorsArray.Get(i)); + if (!parsed_vector_schema) { + ThrowIfNotOk(env, parsed_vector_schema.error()); + return; + } + fields.emplace_back(parsed_vector_schema.value()); + } + } else if (vectorsValue.IsObject()) { + auto parsed_vector_schema = ParseVectorSchema(vectorsValue); if (!parsed_vector_schema) { ThrowIfNotOk(env, parsed_vector_schema.error()); return; } fields.emplace_back(parsed_vector_schema.value()); - } - } else if (vectorsValue.IsObject()) { - auto parsed_vector_schema = ParseVectorSchema(vectorsValue); - if (!parsed_vector_schema) { - ThrowIfNotOk(env, parsed_vector_schema.error()); + } else if (!vectorsValue.IsUndefined() && !vectorsValue.IsNull()) { + ThrowIfNotOk(env, zvec::Status::InvalidArgument( + "CollectionSchema constructor: argument 'vectors' " + "must be an object or an array of objects")); return; } - fields.emplace_back(parsed_vector_schema.value()); - } else { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "CollectionSchema constructor: argument 'vectors' " - "must be an object or an array of objects")); - return; } if (obj.Has("fields")) { @@ -269,7 +265,7 @@ CollectionSchema::CollectionSchema(const Napi::CallbackInfo &info) return; } fields.emplace_back(parsed_field_schema.value()); - } else { + } else if (!fieldsValue.IsUndefined() && !fieldsValue.IsNull()) { ThrowIfNotOk(env, zvec::Status::InvalidArgument( "CollectionSchema constructor: argument 'fields' " "must be an object or an array of objects")); @@ -387,4 +383,4 @@ Napi::Value CollectionSchema::ToString(const Napi::CallbackInfo &info) { } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/schema.h b/src/binding/schema.h index 87f92b0..6666b79 100644 --- a/src/binding/schema.h +++ b/src/binding/schema.h @@ -2,7 +2,7 @@ #include -#include "zvec/db/schema.h" +#include #include "addon.h" @@ -49,4 +49,4 @@ class CollectionSchema : public Napi::ObjectWrap { }; -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/types.cc b/src/binding/types.cc index 905dafa..e002f62 100644 --- a/src/binding/types.cc +++ b/src/binding/types.cc @@ -218,7 +218,9 @@ Napi::Object CreateIndexTypeObject(Napi::Env env) { obj.Set("IVF", static_cast(zvec::IndexType::IVF)); obj.Set("FLAT", static_cast(zvec::IndexType::FLAT)); obj.Set("HNSW_RABITQ", static_cast(zvec::IndexType::HNSW_RABITQ)); + obj.Set("DISKANN", static_cast(zvec::IndexType::DISKANN)); obj.Set("INVERT", static_cast(zvec::IndexType::INVERT)); + obj.Set("FTS", static_cast(zvec::IndexType::FTS)); obj.Freeze(); return obj; } @@ -237,7 +239,9 @@ zvec::Result ParseIndexType(const Napi::Value &value) { case zvec::IndexType::IVF: case zvec::IndexType::FLAT: case zvec::IndexType::HNSW_RABITQ: + case zvec::IndexType::DISKANN: case zvec::IndexType::INVERT: + case zvec::IndexType::FTS: return static_cast(raw); default: return tl::make_unexpected(zvec::Status::InvalidArgument( @@ -317,4 +321,4 @@ Napi::Object InitTypes(Napi::Env env, Napi::Object exports) { } -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/binding/types.h b/src/binding/types.h index b1dfde9..c7b4956 100644 --- a/src/binding/types.h +++ b/src/binding/types.h @@ -2,8 +2,8 @@ #include -#include "zvec/db/status.h" -#include "zvec/db/type.h" +#include +#include namespace binding { @@ -99,4 +99,4 @@ zvec::Result ParseQuantizeType(const Napi::Value &value); Napi::Object InitTypes(Napi::Env env, Napi::Object exports); -} // namespace binding \ No newline at end of file +} // namespace binding diff --git a/src/index.d.mts b/src/index.d.mts index 728e108..e02e1f8 100644 --- a/src/index.d.mts +++ b/src/index.d.mts @@ -1,14 +1,17 @@ export { - isZVecError, ZVecCollectionSchema, + isZVecError, + ZVecCollectionSchema, ZVecCreateAndOpen, ZVecDataType, + ZVecGetDefaultJiebaDictDir, ZVecIndexType, ZVecInitialize, ZVecLogLevel, ZVecLogType, ZVecMetricType, ZVecOpen, - ZVecQuantizeType + ZVecQuantizeType, + ZVecSetDefaultJiebaDictDir } from './index.js'; export type { @@ -17,18 +20,26 @@ export type { ZVecCollection, ZVecCollectionOptions, ZVecCreateIndexOptions, + ZVecDiskAnnIndexParams, + ZVecDiskAnnQueryParams, ZVecDoc, ZVecDocInput, ZVecError, ZVecFieldSchema, ZVecFlatIndexParams, + ZVecFtsIndexParams, + ZVecFtsQuery, + ZVecFtsQueryParams, ZVecHnswIndexParams, ZVecHnswQueryParams, ZVecHnswRabitqIndexParams, - ZVecHnswRabitqQueryParams, ZVecIndexParams, + ZVecHnswRabitqQueryParams, + ZVecIndexParams, ZVecInitOptions, - ZVecInvertIndexParams, ZVecIVFIndexParams, - ZVecIVFQueryParams, ZVecOptimizeOptions, + ZVecInvertIndexParams, + ZVecIVFIndexParams, + ZVecIVFQueryParams, + ZVecOptimizeOptions, ZVecQuery, ZVecQueryParams, ZVecStatus, @@ -45,9 +56,10 @@ declare const _default: { ZVecLogLevel: typeof import('./index.js').ZVecLogLevel; ZVecCollectionSchema: typeof import('./index.js').ZVecCollectionSchema; ZVecInitialize: typeof import('./index.js').ZVecInitialize; + ZVecSetDefaultJiebaDictDir: typeof import('./index.js').ZVecSetDefaultJiebaDictDir; + ZVecGetDefaultJiebaDictDir: typeof import('./index.js').ZVecGetDefaultJiebaDictDir; ZVecCreateAndOpen: typeof import('./index.js').ZVecCreateAndOpen; ZVecOpen: typeof import('./index.js').ZVecOpen; isZVecError: typeof import('./index.js').isZVecError; }; export default _default; - diff --git a/src/index.d.ts b/src/index.d.ts index 4e0c9e8..b4212f1 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -52,7 +52,9 @@ export declare const ZVecIndexType: { readonly IVF: 2; readonly FLAT: 3; readonly HNSW_RABITQ: 4; + readonly DISKANN: 5; readonly INVERT: 10; + readonly FTS: 11; }; export type ZVecIndexType = typeof ZVecIndexType[keyof typeof ZVecIndexType]; @@ -181,18 +183,55 @@ export interface ZVecInitOptions { * @default undefined // Let the system decide */ readonly optimizeThreads?: number; + + /** + * Threshold to switch from inverted-index filtering to forward scan. + * @default undefined + */ + readonly invertToForwardScanRatio?: number; + + /** + * Threshold to use brute-force key lookup over index lookup. + * @default undefined + */ + readonly bruteForceByKeysRatio?: number; + + /** + * Threshold to switch FTS scan strategy for highly selective candidates. + * @default undefined + */ + readonly ftsBruteForceByKeysRatio?: number; + + /** + * Directory containing jieba tokenizer dictionary files. + * @default undefined + */ + readonly jiebaDictDir?: string; } /** * Initializes global Zvec configurations. * This function should be called only once at the start of your application. - * @param options - Optional configuration parameters. + * @param options - Configuration parameters. * * @group Global Configuration */ -export function ZVecInitialize(options?: ZVecInitOptions): void; +export function ZVecInitialize(options: ZVecInitOptions): void; +/** + * Registers the process-wide default jieba dictionary directory. + * + * @group Global Configuration + */ +export function ZVecSetDefaultJiebaDictDir(dir: string): void; + +/** + * Reads the process-wide default jieba dictionary directory. + * + * @group Global Configuration + */ +export function ZVecGetDefaultJiebaDictDir(): string; /** * Base interface for index parameters, requiring the type of index. @@ -259,6 +298,12 @@ export interface ZVecHnswIndexParams extends ZVecIndexParams { * @default ZVecQuantizeType.UNDEFINED */ readonly quantizeType?: ZVecQuantizeType; + + /** + * Whether to allocate HNSW graph nodes in contiguous memory. + * @default false + */ + readonly useContiguousMemory?: boolean; } @@ -349,6 +394,47 @@ export interface ZVecIVFIndexParams extends ZVecIndexParams { } +/** + * Configuration parameters for a DiskANN index on a vector field. + * + * @group Index Parameters + */ +export interface ZVecDiskAnnIndexParams extends ZVecIndexParams { + /** Must be `ZVecIndexType.DISKANN` (5). */ + readonly indexType: typeof ZVecIndexType.DISKANN; + + /** + * Distance metric used for similarity computation. + * @default ZVecMetricType.IP + */ + readonly metricType?: ZVecMetricType; + + /** + * Maximum out-degree of each graph node. + * @default 100 + */ + readonly maxDegree?: number; + + /** + * Candidate list size used during graph construction. + * @default 50 + */ + readonly listSize?: number; + + /** + * Maximum number of PQ chunks for product quantization. `0` lets the engine choose. + * @default 0 + */ + readonly pqChunkNum?: number; + + /** + * Optional quantization type for vector compression. + * @default ZVecQuantizeType.UNDEFINED + */ + readonly quantizeType?: ZVecQuantizeType; +} + + /** * Configuration parameters for an inverted index on a scalar field. * @@ -372,6 +458,35 @@ export interface ZVecInvertIndexParams extends ZVecIndexParams { } +/** + * Configuration parameters for a full-text search index on a scalar string field. + * + * @group Index Parameters + */ +export interface ZVecFtsIndexParams extends ZVecIndexParams { + /** Must be `ZVecIndexType.FTS` (11). */ + readonly indexType: typeof ZVecIndexType.FTS; + + /** + * Tokenizer name. + * @default "standard" + */ + readonly tokenizerName?: string; + + /** + * Token filters. + * @default ["lowercase"] + */ + readonly filters?: string[]; + + /** + * Additional tokenizer parameters as a JSON object string. + * @default "" + */ + readonly extraParams?: string; +} + + /** * Base interface for query-time parameters, requiring the type of index. * @@ -379,7 +494,15 @@ export interface ZVecInvertIndexParams extends ZVecIndexParams { */ export interface ZVecQueryParams { readonly indexType: ZVecIndexType; +} + +/** + * Base interface for vector query-time parameters. + * + * @group Query Parameters + */ +interface ZVecVectorQueryParams extends ZVecQueryParams { /** * Search radius for range queries. Used in combination with top-k to filter results. * @default 0.0 (disabled) @@ -405,7 +528,7 @@ export interface ZVecQueryParams { * * @group Query Parameters */ -export interface ZVecHnswQueryParams extends ZVecQueryParams { +export interface ZVecHnswQueryParams extends ZVecVectorQueryParams { /** Must be `ZVecIndexType.HNSW` (1). */ readonly indexType: typeof ZVecIndexType.HNSW; @@ -424,7 +547,7 @@ export interface ZVecHnswQueryParams extends ZVecQueryParams { * * @group Query Parameters */ -export interface ZVecHnswRabitqQueryParams extends ZVecQueryParams { +export interface ZVecHnswRabitqQueryParams extends ZVecVectorQueryParams { /** Must be `ZVecIndexType.HNSW_RABITQ` (4). */ readonly indexType: typeof ZVecIndexType.HNSW_RABITQ; @@ -441,7 +564,7 @@ export interface ZVecHnswRabitqQueryParams extends ZVecQueryParams { * * @group Query Parameters */ -export interface ZVecIVFQueryParams extends ZVecQueryParams { +export interface ZVecIVFQueryParams extends ZVecVectorQueryParams { /** Must be `ZVecIndexType.IVF` (2). */ readonly indexType: typeof ZVecIndexType.IVF; @@ -453,11 +576,66 @@ export interface ZVecIVFQueryParams extends ZVecQueryParams { } +/** + * Query-time parameters for searches performed against a DiskANN index. + * + * @group Query Parameters + */ +export interface ZVecDiskAnnQueryParams extends ZVecVectorQueryParams { + /** Must be `ZVecIndexType.DISKANN` (5). */ + readonly indexType: typeof ZVecIndexType.DISKANN; + + /** + * Candidate list size used during search. Larger values improve recall but slow down search. + * @default 300 + */ + readonly listSize?: number; +} + + +/** + * Query-time parameters for full-text search. + * + * @group Query Parameters + */ +export interface ZVecFtsQueryParams extends ZVecQueryParams { + /** Must be `ZVecIndexType.FTS` (11). */ + readonly indexType: typeof ZVecIndexType.FTS; + + /** + * Default boolean operator for adjacent bare terms. Supported values are "AND" and "OR"; empty string uses OR. + * @default "" + */ + readonly defaultOperator?: string; +} + + +/** + * Full-text search clause. Provide exactly one of `queryString` or `matchString`. + * + * @group Query Parameters + */ +export interface ZVecFtsQuery { + /** + * Query parser expression. + * @default undefined + */ + readonly queryString?: string; + + /** + * Literal match string. + * @default undefined + */ + readonly matchString?: string; +} + + /** * Query object used to perform searches against a collection. * * You can use it for: * - Vector similarity search: provide both `fieldName` and `vector`. + * - Full-text search: provide both `fieldName` and `fts`. * - Scalar-only filtering: provide only `filter`. * - Hybrid search: provide `fieldName` + `vector` along with a scalar `filter`. * @@ -481,6 +659,12 @@ export interface ZVecQuery { */ vector?: ZVecVector; + /** + * Full-text search clause. Cannot be combined with `vector` in the same query. + * @default undefined + */ + fts?: ZVecFtsQuery; + /** * Boolean expression to pre-filter candidates. * @default undefined @@ -503,7 +687,12 @@ export interface ZVecQuery { * Query-time parameters to fine-tune search behavior. * @default undefined */ - params?: ZVecHnswQueryParams | ZVecHnswRabitqQueryParams | ZVecIVFQueryParams; + params?: + | ZVecHnswQueryParams + | ZVecHnswRabitqQueryParams + | ZVecIVFQueryParams + | ZVecDiskAnnQueryParams + | ZVecFtsQueryParams; } @@ -667,10 +856,10 @@ export interface ZVecFieldSchema { readonly nullable?: boolean; /** - * Optional inverted index parameters for this field. + * Optional scalar index parameters for this field. * @default undefined */ - readonly indexParams?: ZVecInvertIndexParams; + readonly indexParams?: ZVecInvertIndexParams | ZVecFtsIndexParams; } @@ -700,7 +889,12 @@ export interface ZVecVectorSchema { * Vector index parameters for this vector field. * @default undefined */ - readonly indexParams?: ZVecFlatIndexParams | ZVecHnswIndexParams | ZVecHnswRabitqIndexParams | ZVecIVFIndexParams; + readonly indexParams?: + | ZVecFlatIndexParams + | ZVecHnswIndexParams + | ZVecHnswRabitqIndexParams + | ZVecIVFIndexParams + | ZVecDiskAnnIndexParams; } @@ -712,14 +906,14 @@ export interface ZVecVectorSchema { export declare class ZVecCollectionSchema { /** * Creates a new collection schema. - * @param params - An object containing the name, vectors, and optional fields. + * @param params - An object containing the collection name, vector fields, and scalar fields. * @param params.name - The name of the collection. - * @param params.vectors - An array of vector schemas, or a single vector schema object. + * @param params.vectors - An optional array of vector schemas, or a single vector schema object. * @param params.fields - An optional array of scalar field schemas, or a single scalar field schema object. */ constructor(params: { name: string; - vectors: ZVecVectorSchema[] | ZVecVectorSchema; + vectors?: ZVecVectorSchema[] | ZVecVectorSchema; fields?: ZVecFieldSchema[] | ZVecFieldSchema; }); @@ -937,6 +1131,20 @@ export declare class ZVecCollection { */ fetchSync(ids: string | string[]): Record; + /** + * Fetches documents by their IDs. + * @param params - Fetch parameters. + * @param params.ids - A single document ID or an array of document IDs to fetch. + * @param params.outputFields - Scalar fields to include. If undefined, all scalar fields are returned. An empty array returns no scalar fields. + * @param params.includeVector - Whether to include vector data in fetched documents. Defaults to true. + * @returns An object mapping the requested IDs to their corresponding documents. If an ID is not found, it will not be present in the returned object. + */ + fetchSync(params: { + ids: string | string[]; + outputFields?: string[]; + includeVector?: boolean; + }): Record; + /** * Optimizes the collection's internal structures for better performance. * @param options - Optional parameters for the operation. @@ -1015,7 +1223,14 @@ export declare class ZVecCollection { */ createIndexSync(params: { fieldName: string; - indexParams: ZVecFlatIndexParams | ZVecHnswIndexParams | ZVecHnswRabitqIndexParams | ZVecIVFIndexParams | ZVecInvertIndexParams; + indexParams: + | ZVecFlatIndexParams + | ZVecHnswIndexParams + | ZVecHnswRabitqIndexParams + | ZVecIVFIndexParams + | ZVecDiskAnnIndexParams + | ZVecInvertIndexParams + | ZVecFtsIndexParams; indexOptions?: ZVecCreateIndexOptions; }): void; @@ -1045,13 +1260,13 @@ export function ZVecCreateAndOpen( /** - * Opens an existing Zvec collection. - * @param path - The file system path where the collection is stored. - * @param options - Optional parameters for opening the collection. - * @returns An instance of ZVecCollection. - * - * @group Collection - */ + * Opens an existing Zvec collection. + * @param path - The file system path where the collection is stored. + * @param options - Optional parameters for opening the collection. + * @returns An instance of ZVecCollection. + * + * @group Collection + */ export function ZVecOpen( path: string, options?: ZVecCollectionOptions @@ -1067,8 +1282,10 @@ declare const _default: { ZVecLogLevel: typeof ZVecLogLevel; ZVecCollectionSchema: typeof ZVecCollectionSchema; ZVecInitialize: typeof ZVecInitialize; + ZVecSetDefaultJiebaDictDir: typeof ZVecSetDefaultJiebaDictDir; + ZVecGetDefaultJiebaDictDir: typeof ZVecGetDefaultJiebaDictDir; ZVecCreateAndOpen: typeof ZVecCreateAndOpen; ZVecOpen: typeof ZVecOpen; isZVecError: typeof isZVecError; }; -export default _default; \ No newline at end of file +export default _default; diff --git a/src/index.js b/src/index.js index 80d54db..ec9d54a 100644 --- a/src/index.js +++ b/src/index.js @@ -78,7 +78,9 @@ module.exports = { ZVecLogLevel: binding.LogLevel, ZVecCollectionSchema: binding.CollectionSchema, ZVecInitialize: binding.initialize, + ZVecSetDefaultJiebaDictDir: binding.setDefaultJiebaDictDir, + ZVecGetDefaultJiebaDictDir: binding.getDefaultJiebaDictDir, ZVecCreateAndOpen: binding.createAndOpen, ZVecOpen: binding.open, isZVecError, -}; \ No newline at end of file +}; diff --git a/src/index.mjs b/src/index.mjs index 348814a..e0e09dd 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -11,6 +11,8 @@ export const ZVecLogType = cjs.ZVecLogType; export const ZVecLogLevel = cjs.ZVecLogLevel; export const ZVecCollectionSchema = cjs.ZVecCollectionSchema; export const ZVecInitialize = cjs.ZVecInitialize; +export const ZVecSetDefaultJiebaDictDir = cjs.ZVecSetDefaultJiebaDictDir; +export const ZVecGetDefaultJiebaDictDir = cjs.ZVecGetDefaultJiebaDictDir; export const ZVecCreateAndOpen = cjs.ZVecCreateAndOpen; export const ZVecOpen = cjs.ZVecOpen; export const isZVecError = cjs.isZVecError; diff --git a/src/zvec b/src/zvec index d9b0920..da39a33 160000 --- a/src/zvec +++ b/src/zvec @@ -1 +1 @@ -Subproject commit d9b0920ac7426cad85a79e2daeb652e6698ae2bf +Subproject commit da39a33feb3d70a2ffc5ea7e59a1319954e18757 diff --git a/tests/data/helpers.ts b/tests/data/helpers.ts index 222ccda..011bf02 100644 --- a/tests/data/helpers.ts +++ b/tests/data/helpers.ts @@ -44,6 +44,11 @@ const TEST_SCHEMA = { dataType: ZVecDataType.FLOAT, nullable: true, indexParams: { indexType: ZVecIndexType.INVERT, enableRangeOptimization: true } + }, + { + name: 'content', + dataType: ZVecDataType.STRING, + indexParams: { indexType: ZVecIndexType.FTS } } ] }; @@ -57,6 +62,9 @@ export function createTestSchema(name: string): ZVecCollectionSchema { function title(k: number, version: number): string { return `Product_${k}_v${version}`; } function price(k: number, version: number): number { return (k + 0.99) * version; } +function content(k: number, version: number): string { + return `product ${k} version ${version} search content`; +} function dense(k: number, version: number): number[] { const s = k * 1000 + version; return Array.from({ length: DIMENSION }, (_, i) => Math.sin(s * (i + 1))); @@ -70,14 +78,22 @@ export function makeDoc(k: number, fieldVersion: number, vectorVersion: number): return { id: `doc_${k}`, vectors: { dense: dense(k, vectorVersion), sparse: sparse(k, vectorVersion) }, - fields: { title: title(k, fieldVersion), price: price(k, fieldVersion) } + fields: { + title: title(k, fieldVersion), + price: price(k, fieldVersion), + content: content(k, fieldVersion) + } }; } export function makeUpdate(k: number, fieldVersion: number): ZVecDocInput { return { id: `doc_${k}`, - fields: { title: title(k, fieldVersion), price: price(k, fieldVersion) } + fields: { + title: title(k, fieldVersion), + price: price(k, fieldVersion), + content: content(k, fieldVersion) + } }; } diff --git a/tests/data/operations.test.ts b/tests/data/operations.test.ts index c2ab116..2cb8df5 100644 --- a/tests/data/operations.test.ts +++ b/tests/data/operations.test.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import { ZVecCollection, ZVecCreateAndOpen, + ZVecIndexType, ZVecInitialize, ZVecLogLevel, ZVecLogType, @@ -48,6 +49,35 @@ describe('Data Operations Pipeline', () => { }); expectDoc(results[0], 42, 1, 1); }); + + it('should fetch docs correctly', () => { + const doc = makeDoc(42, 1, 1); + expectDoc(collection.fetchSync(doc.id)[doc.id], 42, 1, 1); + expectDoc(collection.fetchSync({ ids: doc.id })[doc.id], 42, 1, 1); + + const fetched = collection.fetchSync({ + ids: doc.id, + outputFields: ['title'], + includeVector: false + })[doc.id]; + expect(fetched.fields.title).toBe(doc.fields!.title); + expect(fetched.fields.price).toBeUndefined(); + expect(Object.keys(fetched.vectors)).toHaveLength(0); + + const docs = [makeDoc(43, 1, 1), makeDoc(44, 1, 1)]; + const ids = docs.map(doc => doc.id); + const batchFetched = collection.fetchSync({ + ids, + outputFields: [] + }); + expect(new Set(Object.keys(batchFetched))).toEqual(new Set(ids)); + for (const doc of docs) { + expect(batchFetched[doc.id].id).toBe(doc.id); + expect(Object.keys(batchFetched[doc.id].fields)).toHaveLength(0); + expect(batchFetched[doc.id].vectors.dense).toBeDefined(); + expect(batchFetched[doc.id].vectors.sparse).toBeDefined(); + } + }); }); @@ -132,6 +162,20 @@ describe('Data Operations Pipeline', () => { filter: `title = "Product_${k}_v1"` }).then(results => ({ k, results })); }); + const ftsQueries = targets.map(k => { + const doc = makeDoc(k, 1, 1); + return collection.query({ + fieldName: 'content', + fts: { matchString: doc.fields!.content }, + params: { + indexType: ZVecIndexType.FTS, + defaultOperator: 'AND' + }, + topk: 10, + outputFields: ['title'], + includeVector: false + }).then(results => ({ k, results })); + }); const denseResults = await Promise.all(denseQueries); for (const { k, results } of denseResults) { @@ -143,6 +187,15 @@ describe('Data Operations Pipeline', () => { expect(results.length).toBe(1); expectDoc(results[0], k, 1, 1); } + const ftsResults = await Promise.all(ftsQueries); + for (const { k, results } of ftsResults) { + const doc = makeDoc(k, 1, 1); + expect(results.length).toBe(1); + expect(results[0].id).toBe(doc.id); + expect(results[0].fields.title).toBe(doc.fields!.title); + expect(results[0].fields.content).toBeUndefined(); + expect(Object.keys(results[0].vectors)).toHaveLength(0); + } }); }); @@ -275,5 +328,22 @@ describe('Data Operations Pipeline', () => { expect(isZVecError(e)).toBe(true); } }); + + it('should throw clear fetch errors for invalid fetch options', () => { + const invalidCalls = [ + () => collection.fetchSync({} as any), + () => collection.fetchSync({ ids: 42 } as any), + () => collection.fetchSync({ ids: 'doc_1', outputFields: 'title' } as any), + () => collection.fetchSync({ ids: 'doc_1', includeVector: 'false' } as any) + ]; + for (const call of invalidCalls) { + try { + call(); + fail('expected to throw'); + } catch (e) { + expect(isZVecError(e)).toBe(true); + } + } + }); }); }); diff --git a/tests/schema.test.ts b/tests/schema.test.ts index c5a143e..2f047ca 100644 --- a/tests/schema.test.ts +++ b/tests/schema.test.ts @@ -1,6 +1,8 @@ import { ZVecCollectionSchema, ZVecDataType, + ZVecDiskAnnIndexParams, + ZVecFtsIndexParams, ZVecHnswIndexParams, ZVecHnswRabitqIndexParams, ZVecIndexType, @@ -251,4 +253,87 @@ describe('CollectionSchema', () => { expect((vectors[1].indexParams as ZVecHnswRabitqIndexParams).efConstruction).toBe(500); expect((vectors[1].indexParams as ZVecHnswRabitqIndexParams).sampleCount).toBe(0); }); -}); \ No newline at end of file + + + (isLinuxX64 ? it : it.skip)('should parse DiskANN index params correctly', () => { + const schema = new ZVecCollectionSchema({ + name: 'test_diskann', + vectors: [ + { + name: 'vector1', + dataType: ZVecDataType.VECTOR_FP32, + dimension: 128, + indexParams: { + indexType: ZVecIndexType.DISKANN, + metricType: ZVecMetricType.L2, + maxDegree: 64, + listSize: 80, + pqChunkNum: 8, + quantizeType: ZVecQuantizeType.FP16 + } + }, + { + name: 'vector2', + dataType: ZVecDataType.VECTOR_FP32, + dimension: 64, + indexParams: { + indexType: ZVecIndexType.DISKANN + } + } + ] + }); + + expect(schema).toBeInstanceOf(ZVecCollectionSchema); + expect(schema.name).toBe('test_diskann'); + + const vectors = schema.vectors(); + expect(vectors.length).toBe(2); + expect(vectors[0].name).toBe('vector1'); + expect(vectors[0].dataType).toBe(ZVecDataType.VECTOR_FP32); + expect(vectors[0].dimension).toBe(128); + expect(vectors[0].indexParams!.indexType).toBe(ZVecIndexType.DISKANN); + expect(vectors[0].indexParams!.metricType).toBe(ZVecMetricType.L2); + expect((vectors[0].indexParams as ZVecDiskAnnIndexParams).maxDegree).toBe(64); + expect((vectors[0].indexParams as ZVecDiskAnnIndexParams).listSize).toBe(80); + expect((vectors[0].indexParams as ZVecDiskAnnIndexParams).pqChunkNum).toBe(8); + expect((vectors[0].indexParams as ZVecDiskAnnIndexParams).quantizeType).toBe(ZVecQuantizeType.FP16); + expect(vectors[1].name).toBe('vector2'); + expect(vectors[1].dataType).toBe(ZVecDataType.VECTOR_FP32); + expect(vectors[1].dimension).toBe(64); + expect(vectors[1].indexParams!.indexType).toBe(ZVecIndexType.DISKANN); + expect(vectors[1].indexParams!.metricType).toBe(ZVecMetricType.IP); + expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).maxDegree).toBe(100); + expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).listSize).toBe(50); + expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).pqChunkNum).toBe(0); + expect((vectors[1].indexParams as ZVecDiskAnnIndexParams).quantizeType).toBe(ZVecQuantizeType.UNDEFINED); + }); + + + it('should parse FTS index params correctly', () => { + const schema = new ZVecCollectionSchema({ + name: 'test_fts', + fields: { + name: 'content', + dataType: ZVecDataType.STRING, + indexParams: { + indexType: ZVecIndexType.FTS, + tokenizerName: 'whitespace', + filters: ['lowercase'], + extraParams: '{"jieba_dict_dir": "/path/to/jieba/dict"}' + } + } + }); + + expect(schema).toBeInstanceOf(ZVecCollectionSchema); + expect(schema.name).toBe('test_fts'); + + const fields = schema.fields(); + expect(fields.length).toBe(1); + expect(fields[0].name).toBe('content'); + expect(fields[0].dataType).toBe(ZVecDataType.STRING); + expect(fields[0].indexParams!.indexType).toBe(ZVecIndexType.FTS); + expect((fields[0].indexParams as ZVecFtsIndexParams).tokenizerName).toBe('whitespace'); + expect((fields[0].indexParams as ZVecFtsIndexParams).filters).toEqual(['lowercase']); + expect((fields[0].indexParams as ZVecFtsIndexParams).extraParams).toBe('{"jieba_dict_dir": "/path/to/jieba/dict"}'); + }); +});