feat: add fts and diskann index

.github/workflows/publish.yml

@@ -39,6 +39,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
+          dnf install -y libaio-devel
           curl -sL ${{ matrix.cmake_url }} | tar xz -C /usr/local --strip-components=1
           npm install --ignore-scripts
 

.github/workflows/test-source.yml

@@ -35,6 +35,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
+          dnf install -y libaio-devel
           curl -sL https://cmake.org/files/v3.27/cmake-3.27.9-linux-x86_64.tar.gz | tar xz -C /usr/local --strip-components=1
           npm install --ignore-scripts
 

CMakeLists.txt

@@ -68,6 +68,10 @@ set(ZVEC_BUILD_TARGETS
     --target zvec_turbo
     --target zvec_db
 )
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")
+    list(APPEND ZVEC_BUILD_TARGETS --target core_knn_diskann)
+    list(APPEND ZVEC_BYPRODUCTS ${ZVEC_LIB_DIR}/libzvec_diskann_plugin.so)
+endif()
 set(ZVEC_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release)
 if(MSVC)
     list(APPEND ZVEC_CMAKE_ARGS
@@ -141,6 +145,7 @@ if (NOT WIN32)
         ${GLOG_LIB}
         ${GFLAGS_LIB}
         ${PROTOBUF_LIB}
+        FastPFOR
         lz4
     )
 else ()
@@ -165,6 +170,7 @@ else ()
         ${GLOG_LIB}
         ${GFLAGS_LIB}
         ${PROTOBUF_LIB}
+        FastPFOR
         lz4
         rpcrt4
         shlwapi
@@ -318,3 +324,12 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
         )
     endif()
 endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64")
+    add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "${ZVEC_LIB_DIR}/libzvec_diskann_plugin.so"
+            "$<TARGET_FILE_DIR:${PROJECT_NAME}>/libzvec_diskann_plugin.so"
+        COMMENT "Copying DiskANN runtime plugin"
+    )
+endif()

package.json

@@ -42,6 +42,7 @@
     "src/index.d.mts",
     "scripts/install.js",
     "zvec_node_binding.node",
+    "libzvec_diskann_plugin.so",
     "README.md",
     "LICENSE"
   ],
@@ -79,4 +80,4 @@
   "dependencies": {
     "bindings": "^1.5.0"
   }
-}
+}

packages/bindings-linux-x64/package.json

@@ -10,6 +10,7 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node"
+    "zvec_node_binding.node",
+    "libzvec_diskann_plugin.so"
   ]
-}
+}

scripts/build.js

@@ -56,8 +56,18 @@ try {
   const targetPath = path.join(platformPackageDir, 'zvec_node_binding.node');
   fs.copyFileSync(binaryPath, targetPath);
 
+  // DiskANN currently ships as a runtime-loaded plugin. Keep it next to the
+  // addon binary so the C++ engine can auto-load it when DiskANN is used.
+  const diskAnnPluginPath = path.join(BUILD_TARGET_DIR, 'libzvec_diskann_plugin.so');
+  if (fs.existsSync(diskAnnPluginPath)) {
+    fs.copyFileSync(
+      diskAnnPluginPath,
+      path.join(platformPackageDir, 'libzvec_diskann_plugin.so')
+    );
+  }
+
   console.log(`✅ Binary compiled and packaged for ${platformPackageName} at: ${targetPath}`);
 } catch (error) {
   console.error('❌ Error during build and packaging:', error.message);
   process.exit(1);
-}
+}

scripts/pack-local.js

@@ -26,6 +26,15 @@ try {
   fs.copyFileSync(targetPath, destPath);
   console.log(`Binary copied from ${targetPath} to ${destPath}`);
 
+  // DiskANN currently ships as a runtime-loaded plugin. Local packs bundle it
+  // beside the addon binary so the C++ engine can auto-load it when needed.
+  const pluginPath = path.join(platformPackageDir, 'libzvec_diskann_plugin.so');
+  if (fs.existsSync(pluginPath)) {
+    const pluginDestPath = path.join(PACKAGE_ROOT, 'libzvec_diskann_plugin.so');
+    fs.copyFileSync(pluginPath, pluginDestPath);
+    console.log(`DiskANN plugin copied from ${pluginPath} to ${pluginDestPath}`);
+  }
+
   // Temporarily remove optionalDependencies from package.json
   // (local pack bundles the binary directly, no need for platform packages)
   const packageJsonPath = path.join(PACKAGE_ROOT, 'package.json');
@@ -44,4 +53,4 @@ try {
 } catch (error) {
   console.error('❌ Error during build and packaging:', error.message);
   process.exit(1);
-}
+}

src/binding/addon.cc

@@ -39,4 +39,4 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) {
 }
 
 
-NODE_API_MODULE(zvec_node_binding, Init)
+NODE_API_MODULE(zvec_node_binding, Init)

src/binding/async_workers.cc

@@ -31,7 +31,7 @@ void DeleteByFilterWorker::OnError(const Napi::Error &error) {
 
 QueryWorker::QueryWorker(Napi::Env env, zvec::Collection::Ptr collection,
                          zvec::CollectionSchema::Ptr schema,
-                         zvec::VectorQuery query,
+                         zvec::SearchQuery query,
                          Napi::Promise::Deferred deferred)
     : Napi::AsyncWorker(env),
       collection_(collection),

src/binding/async_workers.h

@@ -3,8 +3,8 @@
 
 #include <napi.h>
 #include <string>
-#include "zvec/db/collection.h"
-#include "zvec/db/status.h"
+#include <zvec/db/collection.h>
+#include <zvec/db/status.h>
 
 
 namespace binding {
@@ -30,7 +30,7 @@ class DeleteByFilterWorker : public Napi::AsyncWorker {
 class QueryWorker : public Napi::AsyncWorker {
  public:
   QueryWorker(Napi::Env env, zvec::Collection::Ptr collection,
-              zvec::CollectionSchema::Ptr schema, zvec::VectorQuery query,
+              zvec::CollectionSchema::Ptr schema, zvec::SearchQuery query,
               Napi::Promise::Deferred deferred);
 
   void Execute() override;
@@ -40,7 +40,7 @@ class QueryWorker : public Napi::AsyncWorker {
  private:
   zvec::Collection::Ptr collection_;
   zvec::CollectionSchema::Ptr schema_;
-  zvec::VectorQuery query_;
+  zvec::SearchQuery query_;
   Napi::Promise::Deferred deferred_;
   zvec::Status status_;
   zvec::DocPtrList results_;

src/binding/collection.cc

@@ -1,4 +1,5 @@
 #include "collection.h"
+#include <optional>
 #include "async_workers.h"
 #include "doc.h"
 #include "params.h"
@@ -570,7 +571,7 @@ Napi::Value Collection::Query(const Napi::CallbackInfo &info) {
     return env.Undefined();
   }
 
-  if (auto parsed_query = ParseVectorQuery(info[0], get_wrapped_schema());
+  if (auto parsed_query = ParseSearchQuery(info[0], get_wrapped_schema());
       parsed_query) {
     auto res = collection_->Query(parsed_query.value());
     if (res) {
@@ -604,7 +605,7 @@ Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) {
     return deferred.Promise();
   }
 
-  if (auto parsed_query = ParseVectorQuery(info[0], get_wrapped_schema());
+  if (auto parsed_query = ParseSearchQuery(info[0], get_wrapped_schema());
       parsed_query) {
     auto *worker = new QueryWorker(env, collection_, get_wrapped_schema(),
                                    std::move(parsed_query.value()), deferred);
@@ -620,42 +621,96 @@ Napi::Value Collection::QueryAsync(const Napi::CallbackInfo &info) {
 Napi::Value Collection::Fetch(const Napi::CallbackInfo &info) {
   Napi::Env env = info.Env();
   if (ThrowIfClosed(env)) return env.Undefined();
-  if (info.Length() != 1) {
-    ThrowIfNotOk(
-        env, zvec::Status::InvalidArgument(
-                 "Collection.fetch(): Expected exactly 1 argument. "
-                 "Argument must be an id string or an array of id strings."));
+  if (info.Length() != 1 ||
+      !(info[0].IsObject() || info[0].IsString() || info[0].IsArray())) {
+    ThrowIfNotOk(env, zvec::Status::InvalidArgument(
+                          "Collection.fetch(): Expected exactly 1 argument: "
+                          "(ids: string | string[]) or "
+                          "({ ids: string | string[], outputFields?: string[], "
+                          "includeVector?: boolean })."));
     return env.Undefined();
   }
 
-  std::vector<std::string> pks{};
   zvec::Result<zvec::DocPtrMap> res;
-
-  if (info[0].IsArray()) {
-    Napi::Array pkArray = info[0].As<Napi::Array>();
-    uint32_t length = pkArray.Length();
-    pks.reserve(length);
-    for (uint32_t i = 0; i < length; i++) {
-      if (pkArray.Get(i).IsString()) {
-        pks.emplace_back(pkArray.Get(i).As<Napi::String>().Utf8Value());
-      } else {
-        ThrowIfNotOk(env,
-                     zvec::Status::InvalidArgument("Expected a string for id"));
-        return env.Undefined();
+  std::vector<std::string> pks{};
+  std::optional<std::vector<std::string>> output_fields{};
+  bool include_vector{true};
+
+  auto parse_ids = [&](const Napi::Value &ids) -> bool {
+    if (ids.IsArray()) {
+      Napi::Array pkArray = ids.As<Napi::Array>();
+      uint32_t length = pkArray.Length();
+      pks.reserve(length);
+      for (uint32_t i = 0; i < length; i++) {
+        if (pkArray.Get(i).IsString()) {
+          pks.emplace_back(pkArray.Get(i).As<Napi::String>().Utf8Value());
+        } else {
+          ThrowIfNotOk(env, zvec::Status::InvalidArgument(
+                                "Collection.fetch(): Expected a string or array "
+                                "of strings for 'ids'"));
+          return false;
+        }
       }
+      return true;
+    }
+    if (ids.IsString()) {
+      pks.emplace_back(ids.As<Napi::String>().Utf8Value());
+      return true;
     }
-    res = collection_->Fetch(pks);
+    ThrowIfNotOk(env, zvec::Status::InvalidArgument(
+                          "Collection.fetch(): Expected a string or array of "
+                          "strings for 'ids'"));
+    return false;
+  };
+
+  if (info[0].IsString() || info[0].IsArray()) {
+    if (!parse_ids(info[0])) return env.Undefined();
   } else {
-    if (info[0].IsString()) {
-      pks.emplace_back(info[0].As<Napi::String>().Utf8Value());
-    } else {
+    auto obj = info[0].As<Napi::Object>();
+    if (!obj.Has("ids")) {
       ThrowIfNotOk(env,
-                   zvec::Status::InvalidArgument("Expected a string for id"));
+                   zvec::Status::InvalidArgument(
+                       "Collection.fetch(): Missing required argument 'ids'"));
       return env.Undefined();
     }
-    res = collection_->Fetch(pks);
+    if (obj.Has("outputFields") && !obj.Get("outputFields").IsUndefined()) {
+      if (!obj.Get("outputFields").IsArray()) {
+        ThrowIfNotOk(env, zvec::Status::InvalidArgument(
+                              "Collection.fetch(): argument 'outputFields' must "
+                              "be an array of strings"));
+        return env.Undefined();
+      }
+      std::vector<std::string> fields{};
+      auto array = obj.Get("outputFields").As<Napi::Array>();
+      fields.reserve(array.Length());
+      for (uint32_t i = 0; i < array.Length(); i++) {
+        auto item = array.Get(i);
+        if (!item.IsString()) {
+          ThrowIfNotOk(env, zvec::Status::InvalidArgument(
+                                "Collection.fetch(): Expected an array of "
+                                "strings for 'outputFields'"));
+          return env.Undefined();
+        }
+        fields.emplace_back(item.As<Napi::String>().Utf8Value());
+      }
+      output_fields = std::move(fields);
+    }
+    if (obj.Has("includeVector") && !obj.Get("includeVector").IsUndefined()) {
+      if (!obj.Get("includeVector").IsBoolean()) {
+        ThrowIfNotOk(
+            env,
+            zvec::Status::InvalidArgument(
+                "Collection.fetch(): Expected a boolean for 'includeVector'"));
+        return env.Undefined();
+      }
+      include_vector = obj.Get("includeVector").As<Napi::Boolean>().Value();
+    }
+
+    if (!parse_ids(obj.Get("ids"))) return env.Undefined();
   }
 
+  res = collection_->Fetch(pks, output_fields, include_vector);
+
   if (res) {
     Napi::Object obj = Napi::Object::New(env);
     const zvec::DocPtrMap &map = res.value();
@@ -1006,4 +1061,4 @@ Napi::Value Collection::DropIndex(const Napi::CallbackInfo &info) {
 }
 
 
-}  // namespace binding
+}  // namespace binding

src/binding/collection.h

@@ -2,7 +2,7 @@
 
 
 #include <napi.h>
-#include "zvec/db/collection.h"
+#include <zvec/db/collection.h>
 #include "addon.h"
 
 
@@ -85,4 +85,4 @@ class Collection : public Napi::ObjectWrap<Collection> {
 };
 
 
-}  // namespace binding
+}  // namespace binding