Added PQ quantizer in Turbo

.gitignore

@@ -60,9 +60,9 @@ allure-*
 !build_android.sh
 !build_ios.sh
 
-# congfig
+# config
 doc/
 config/
 examples/python/
 examples/c_api/
-logs/
+logs/

examples/c++/CMakeLists.txt

@@ -14,7 +14,10 @@ if(NOT DEFINED HOST_BUILD_DIR)
 endif()
 
 get_filename_component(ZVEC_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE)
-set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include)
+# Include both src/include (public headers) and src (internal headers that
+# are transitively reachable from public headers, e.g.
+# turbo/quantizer/quantizer.h via index_factory.h).
+set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include ${ZVEC_ROOT_DIR}/src)
 set(ZVEC_LIB_DIR ${ZVEC_ROOT_DIR}/${HOST_BUILD_DIR}/lib)
 
 # Add include and library search paths

src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h

@@ -73,7 +73,7 @@ struct HnswSparseHeader {
 
 struct SparseData {
  public:
-  SparseData() {};
+  SparseData(){};
 
   SparseData(uint32_t sparse_count, const uint32_t *sparse_indices,
              const void *sparse_vec)

src/core/framework/index_factory.cc

@@ -257,5 +257,18 @@ std::vector<std::string> IndexFactory::AllRefiners(void) {
   return ailego::Factory<IndexRefiner>::Classes();
 }
 
+std::shared_ptr<turbo::Quantizer> IndexFactory::CreateQuantizer(
+    const std::string &name) {
+  return ailego::Factory<zvec::turbo::Quantizer>::MakeShared(name.c_str());
+}
+
+bool IndexFactory::HasQuantizer(const std::string &name) {
+  return ailego::Factory<turbo::Quantizer>::Has(name.c_str());
+}
+
+std::vector<std::string> IndexFactory::AllQuantizers(void) {
+  return ailego::Factory<turbo::Quantizer>::Classes();
+}
+
 }  // namespace core
 }  // namespace zvec

src/core/framework/index_meta.cc

@@ -30,7 +30,8 @@ struct IndexMetaFormatHeader {
   uint32_t space_id;
   uint32_t attachment_offset;
   uint32_t attachment_size;
-  uint8_t reserved_[4092];
+  uint32_t extra_meta_size;
+  uint8_t reserved_[4088];
 };
 
 static_assert(sizeof(IndexMetaFormatHeader) % 32 == 0,
@@ -47,6 +48,7 @@ void IndexMeta::serialize(std::string *out) const {
   format.dimension = dimension_;
   format.unit_size = unit_size_;
   format.space_id = space_id_;
+  format.extra_meta_size = extra_meta_size_;
 
   if (!metric_name_.empty()) {
     ailego::Params item;

src/core/utility/visit_filter.h

@@ -44,7 +44,7 @@ class VisitBloomFilter {
   static constexpr int N = 5;
   struct Context {
     Context()
-        : mt(std::chrono::system_clock::now().time_since_epoch().count()) {};
+        : mt(std::chrono::system_clock::now().time_since_epoch().count()){};
     VisitFilterHeader h;
     std::mt19937 mt;
     ailego::BloomFilter<N> *filter{nullptr};
@@ -367,7 +367,7 @@ class VisitFilter {
     ByteMap = VisitByteMap::mode
   };
 
-  VisitFilter() : mode_(0), ctx_(nullptr) {};
+  VisitFilter() : mode_(0), ctx_(nullptr){};
 
   inline bool visited(id_t idx) {
     PROXIMA_HNSW_VISITFILTER_CALL_IMPL(visited, idx);

src/db/index/column/inverted_column/inverted_column_indexer.h

@@ -68,7 +68,7 @@ class InvertedColumnIndexer {
         field_(field),
         path_(context.db_path_),
         ctx_(context),
-        read_only_(read_only) {};
+        read_only_(read_only){};
 
   InvertedColumnIndexer(const InvertedColumnIndexer &) = delete;
   InvertedColumnIndexer(InvertedColumnIndexer &&) = delete;

src/db/index/column/inverted_column/inverted_indexer.h

@@ -33,7 +33,7 @@ class InvertedIndexer {
                            const std::vector<FieldSchema> &fields)
       : collection_name_(collection_name),
         working_dir_(working_dir),
-        fields_(fields) {};
+        fields_(fields){};
 
 
   virtual ~InvertedIndexer() {

src/db/index/column/inverted_column/inverted_search_result.h

@@ -34,7 +34,7 @@ class InvertedSearchResult
   class Filter : public IndexFilter {
    public:
     explicit Filter(std::shared_ptr<const InvertedSearchResult> result)
-        : result_(std::move(result)) {};
+        : result_(std::move(result)){};
 
     bool is_filtered(uint64_t id) const override {
       return !result_->contains(id);

src/db/index/common/delete_store.h

@@ -29,7 +29,7 @@ class DeleteStore : public std::enable_shared_from_this<DeleteStore> {
   using Ptr = std::shared_ptr<DeleteStore>;
 
   explicit DeleteStore(std::string collection_name)
-      : collection_name_(std::move(collection_name)) {};
+      : collection_name_(std::move(collection_name)){};
 
   ~DeleteStore() {
     LOG_INFO("Closed delete store");

src/db/index/common/id_map.h

@@ -30,7 +30,7 @@ class IDMap {
   using Ptr = std::shared_ptr<IDMap>;
 
   explicit IDMap(std::string collection_name)
-      : collection_name_(std::move(collection_name)) {};
+      : collection_name_(std::move(collection_name)){};
 
   ~IDMap() {
     if (opened_) {

src/db/index/common/index_params.cc

@@ -26,8 +26,7 @@ namespace zvec {
 
 std::string InvertIndexParams::to_string() const {
   std::ostringstream oss;
-  oss << "InvertIndexParams{"
-      << "enable_range_optimization:"
+  oss << "InvertIndexParams{" << "enable_range_optimization:"
       << (enable_range_optimization_ ? "true" : "false")
       << ", enable_extended_wildcard:"
       << (enable_extended_wildcard_ ? "true" : "false") << "}";

src/db/index/common/meta.h

@@ -124,8 +124,8 @@ class BlockMeta {
 
   std::string to_string() const {
     std::ostringstream oss;
-    oss << "BlockMeta{"
-        << "id:" << id_ << ",type:" << BlockTypeCodeBook::AsString(type_)
+    oss << "BlockMeta{" << "id:" << id_
+        << ",type:" << BlockTypeCodeBook::AsString(type_)
         << ",min_doc_id:" << min_doc_id_ << ",max_doc_id:" << max_doc_id_
         << ",doc_count:" << doc_count_ << ",columns:[";
 
@@ -179,7 +179,7 @@ class SegmentMeta {
   using Ptr = std::shared_ptr<SegmentMeta>;
 
  public:
-  SegmentMeta() {};
+  SegmentMeta(){};
 
   explicit SegmentMeta(SegmentID id) : id_(id) {}
 
@@ -368,8 +368,7 @@ class SegmentMeta {
 
   std::string to_string() const {
     std::ostringstream oss;
-    oss << "SegmentMeta{"
-        << "id:" << id_ << ",persisted_blocks:[";
+    oss << "SegmentMeta{" << "id:" << id_ << ",persisted_blocks:[";
 
     for (size_t i = 0; i < persisted_blocks_.size(); ++i) {
       if (i > 0) oss << ",";

src/db/index/common/schema.cc

@@ -296,8 +296,7 @@ Status FieldSchema::validate() const {
 
 std::string FieldSchema::to_string() const {
   std::ostringstream oss;
-  oss << "FieldSchema{"
-      << "name:'" << name_ << "'"
+  oss << "FieldSchema{" << "name:'" << name_ << "'"
       << ",data_type:" << DataTypeCodeBook::AsString(data_type_)
       << ",nullable:" << (nullable_ ? "true" : "false")
       << ",dimension:" << dimension_;
@@ -382,8 +381,7 @@ Status CollectionSchema::validate() const {
 
 std::string CollectionSchema::to_string() const {
   std::ostringstream oss;
-  oss << "CollectionSchema{"
-      << "name:'" << name_ << "'"
+  oss << "CollectionSchema{" << "name:'" << name_ << "'"
       << ",max_doc_count_per_segment:" << max_doc_count_per_segment_
       << ",fields:[";
 

src/db/index/common/stats.cc

@@ -19,8 +19,8 @@
 namespace zvec {
 std::string CollectionStats::to_string() const {
   std::ostringstream oss;
-  oss << "CollectionStats{"
-      << "doc_count:" << doc_count << ",index_completeness:{";
+  oss << "CollectionStats{" << "doc_count:" << doc_count
+      << ",index_completeness:{";
 
   size_t i = 0;
   for (const auto &pair : index_completeness) {

src/db/sqlengine/parser/zvec_parser.h

@@ -26,7 +26,7 @@ class ZVecParser {
   using Ptr = std::shared_ptr<ZVecParser>;
 
   ZVecParser() = default;
-  virtual ~ZVecParser() {};
+  virtual ~ZVecParser(){};
 
   virtual SQLInfo::Ptr parse(const std::string &query,
                              bool formatted_tree = false) = 0;

src/include/zvec/ailego/pattern/closure.h

@@ -309,7 +309,7 @@ class Callback : public Callback<void> {
 
  protected:
   //! Constructor
-  Callback(void) {};
+  Callback(void){};
 };
 
 /*! Callback Implementation

src/include/zvec/core/framework/index_factory.h

@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <turbo/quantizer/quantizer.h>
 #include <zvec/ailego/pattern/factory.h>
 #include <zvec/core/framework/index_builder.h>
 #include <zvec/core/framework/index_cluster.h>
@@ -167,6 +168,16 @@ struct IndexFactory {
 
   //! Retrieve all refiner classes
   static std::vector<std::string> AllRefiners(void);
+
+  //! Create a quantizer by name
+  static std::shared_ptr<zvec::turbo::Quantizer> CreateQuantizer(
+      const std::string &name);
+
+  //! Test if the quantizer exists
+  static bool HasQuantizer(const std::string &name);
+
+  //! Retrieve all quantizer classes
+  static std::vector<std::string> AllQuantizers(void);
 };
 
 //! Register Index Metric
@@ -283,5 +294,13 @@ struct IndexFactory {
 #define INDEX_FACTORY_REGISTER_REFINER(__IMPL__, ...) \
   INDEX_FACTORY_REGISTER_REFINER_ALIAS(__IMPL__, __IMPL__, ##__VA_ARGS__)
 
+//! Register Quantizer
+#define INDEX_FACTORY_REGISTER_QUANTIZER_ALIAS(__NAME__, __IMPL__, ...) \
+  AILEGO_FACTORY_REGISTER(__NAME__, turbo::Quantizer, __IMPL__, ##__VA_ARGS__)
+
+//! Register Quantizer
+#define INDEX_FACTORY_REGISTER_QUANTIZER(__IMPL__, ...) \
+  INDEX_FACTORY_REGISTER_QUANTIZER_ALIAS(__IMPL__, __IMPL__, ##__VA_ARGS__)
+
 }  // namespace core
 }  // namespace zvec