From a0369b8e2345cf01c2bcb73108ed47514a4e8361 Mon Sep 17 00:00:00 2001 From: "jiliang.ljl" Date: Tue, 14 Jul 2026 20:50:55 +0800 Subject: [PATCH] feat(fts): add ngram tokenizer --- python/zvec/model/param/__init__.pyi | 18 +- .../python/model/param/python_param.cc | 16 +- src/db/index/column/fts_column/fts_types.h | 2 +- .../tokenizer/standard_tokenizer.cc | 213 ++++++++++++++++++ .../fts_column/tokenizer/standard_tokenizer.h | 32 +++ .../fts_column/tokenizer/tokenizer_factory.cc | 2 + src/include/zvec/c_api.h | 8 +- src/include/zvec/db/index_params.h | 8 +- .../column/fts_column/ngram_tokenizer_test.cc | 187 +++++++++++++++ 9 files changed, 479 insertions(+), 7 deletions(-) create mode 100644 tests/db/index/column/fts_column/ngram_tokenizer_test.cc diff --git a/python/zvec/model/param/__init__.pyi b/python/zvec/model/param/__init__.pyi index 2653d2c80..b60f81245 100644 --- a/python/zvec/model/param/__init__.pyi +++ b/python/zvec/model/param/__init__.pyi @@ -714,8 +714,8 @@ class FtsIndexParam(IndexParam): Attributes: type (IndexType): Always ``IndexType.FTS``. - tokenizer_name (str): Name of the tokenizer (one of "standard", "jieba", - "whitespace"). + tokenizer_name (str): Name of the tokenizer (one of "standard", "ngram", + "jieba", "whitespace"). Default is "standard". filters (list[str]): List of token filter names applied after tokenization. Supported filters are "lowercase", "ascii_folding", and "stemmer". @@ -725,6 +725,13 @@ class FtsIndexParam(IndexParam): Tokenizers: standard: - "max_token_length" (positive integer). + ngram: + - "ngram_min" (positive integer, default 2). + - "ngram_max" (positive integer, default 2). + - "token_chars" (array of "letter", "digit", + "whitespace", "punctuation", "symbol"; default [] keeps + all valid UTF-8 characters). custom_token_chars is not + supported. jieba: - "jieba_dict_dir" (directory containing jieba.dict.utf8 and hmm_model.utf8). @@ -772,6 +779,13 @@ class FtsIndexParam(IndexParam): Tokenizers: standard: - "max_token_length" (positive integer). + ngram: + - "ngram_min" (positive integer, default 2). + - "ngram_max" (positive integer, default 2). + - "token_chars" (array of "letter", "digit", + "whitespace", "punctuation", "symbol"; default [] + keeps all valid UTF-8 characters). custom_token_chars + is not supported. jieba: - "jieba_dict_dir". - "user_dict_path". diff --git a/src/binding/python/model/param/python_param.cc b/src/binding/python/model/param/python_param.cc index 3c3ff623b..3e9376223 100644 --- a/src/binding/python/model/param/python_param.cc +++ b/src/binding/python/model/param/python_param.cc @@ -257,8 +257,8 @@ Controls the tokenizer pipeline used during indexing and querying. Attributes: type (IndexType): Always ``IndexType.FTS``. - tokenizer_name (str): Name of the tokenizer (one of "standard", "jieba", - "whitespace"). + tokenizer_name (str): Name of the tokenizer (one of "standard", "ngram", + "jieba", "whitespace"). Default is "standard". filters (list[str]): List of token filter names applied after tokenization. Supported values include "lowercase", "ascii_folding", and "stemmer". @@ -268,6 +268,12 @@ Controls the tokenizer pipeline used during indexing and querying. Tokenizers: standard: - "max_token_length" (positive integer). + ngram: + - "ngram_min" (positive integer, default 2). + - "ngram_max" (positive integer, default 2). + - "token_chars" (array of "letter", "digit", "whitespace", + "punctuation", "symbol"; default [] keeps all valid UTF-8 + characters). custom_token_chars is not supported. jieba: - "jieba_dict_dir" (directory containing jieba.dict.utf8 and hmm_model.utf8). @@ -316,6 +322,12 @@ Constructs an FtsIndexParam instance. Tokenizers: standard: - "max_token_length" (positive integer). + ngram: + - "ngram_min" (positive integer, default 2). + - "ngram_max" (positive integer, default 2). + - "token_chars" (array of "letter", "digit", "whitespace", + "punctuation", "symbol"; default [] keeps all valid UTF-8 + characters). custom_token_chars is not supported. jieba: - "jieba_dict_dir". - "user_dict_path". diff --git a/src/db/index/column/fts_column/fts_types.h b/src/db/index/column/fts_column/fts_types.h index 46eee4657..483e42c40 100644 --- a/src/db/index/column/fts_column/fts_types.h +++ b/src/db/index/column/fts_column/fts_types.h @@ -51,7 +51,7 @@ struct FtsSegmentStats { }; struct FtsIndexParams { - // Supported tokenizers: standard, jieba, whitespace. + // Supported tokenizers: standard, ngram, jieba, whitespace. std::string tokenizer_name{"standard"}; // Supported filters: lowercase, ascii_folding. std::vector filters{"lowercase"}; diff --git a/src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc b/src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc index fc612e805..545ee0777 100644 --- a/src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc +++ b/src/db/index/column/fts_column/tokenizer/standard_tokenizer.cc @@ -14,7 +14,9 @@ #include "standard_tokenizer.h" #include +#include #include +#include #include namespace zvec::fts { @@ -24,6 +26,7 @@ namespace { constexpr uint32_t kDefaultMaxTokenLength = 255; constexpr uint32_t kMinMaxTokenLength = 1; constexpr uint32_t kMaxMaxTokenLength = 1048576; +constexpr uint32_t kDefaultNGramLength = 2; constexpr uint32_t kVariationSelector16 = 0xFE0F; constexpr uint32_t kKeycap = 0x20E3; constexpr uint32_t kUnicodeMaxCodepoint = 0x10FFFF; @@ -32,6 +35,11 @@ constexpr size_t kUnicodePageCount = (kUnicodeMaxCodepoint >> kUnicodePageShift) + 1; constexpr size_t kCodepointCacheSize = 1024; constexpr size_t kMaxInitialTokenCapacity = 4096; +constexpr uint32_t kNGramTokenCharLetter = 1u << 0; +constexpr uint32_t kNGramTokenCharDigit = 1u << 1; +constexpr uint32_t kNGramTokenCharWhitespace = 1u << 2; +constexpr uint32_t kNGramTokenCharPunctuation = 1u << 3; +constexpr uint32_t kNGramTokenCharSymbol = 1u << 4; enum class WordBreakClass : uint8_t { Other, @@ -65,6 +73,7 @@ struct Codepoint { uint32_t start{0}; uint32_t end{0}; WordBreakClass cls{WordBreakClass::Other}; + bool valid{false}; bool extended_pictographic{false}; bool emoji_modifier_base{false}; bool emoji_modifier{false}; @@ -99,6 +108,11 @@ struct UnicodeRangeIndex { uint32_t last{0}; }; +struct CodepointSpan { + uint32_t start{0}; + uint32_t end{0}; +}; + template struct UnicodeRangeIndexTable { std::array pages; @@ -492,6 +506,7 @@ std::vector decode_utf8(const std::string &text) { item.cp = cp; item.start = static_cast(index); item.end = static_cast(index + bytes); + item.valid = true; uint32_t codepoint = static_cast(cp); CodepointProperties props = get_codepoint_properties(codepoint, cache); item.cls = props.cls; @@ -987,6 +1002,174 @@ std::vector tokenize_ascii(const std::string &text, return tokens; } +bool parse_positive_uint32_param(const ailego::JsonObject &config, + const char *key, uint32_t default_value, + uint32_t *value) { + *value = default_value; + auto json_value = config[key]; + if (json_value.is_null()) { + return true; + } + if (!json_value.is_integer()) { + LOG_ERROR("NGramTokenizer: %s must be integer", key); + return false; + } + int64_t configured_value = json_value.as_integer(); + if (configured_value <= 0 || + configured_value > + static_cast(std::numeric_limits::max())) { + LOG_ERROR("NGramTokenizer: %s must be positive uint32", key); + return false; + } + *value = static_cast(configured_value); + return true; +} + +bool parse_ngram_token_char(const std::string &token_char, uint32_t *mask) { + if (token_char == "letter") { + *mask |= kNGramTokenCharLetter; + return true; + } + if (token_char == "digit") { + *mask |= kNGramTokenCharDigit; + return true; + } + if (token_char == "whitespace") { + *mask |= kNGramTokenCharWhitespace; + return true; + } + if (token_char == "punctuation") { + *mask |= kNGramTokenCharPunctuation; + return true; + } + if (token_char == "symbol") { + *mask |= kNGramTokenCharSymbol; + return true; + } + LOG_ERROR("NGramTokenizer: unsupported token_chars value: %s", + token_char.c_str()); + return false; +} + +bool parse_ngram_token_chars(const ailego::JsonObject &config, uint32_t *mask) { + *mask = 0; + auto token_chars_val = config["token_chars"]; + if (token_chars_val.is_null()) { + return true; + } + if (!token_chars_val.is_array()) { + LOG_ERROR("NGramTokenizer: token_chars must be an array"); + return false; + } + + const auto &token_chars = token_chars_val.as_array(); + for (const auto &token_char_val : token_chars) { + if (!token_char_val.is_string()) { + LOG_ERROR("NGramTokenizer: token_chars entries must be strings"); + return false; + } + if (!parse_ngram_token_char(token_char_val.as_stl_string(), mask)) { + return false; + } + } + return true; +} + +bool is_ngram_whitespace(uint32_t codepoint, utf8proc_category_t category) { + if ((codepoint >= 0x0009 && codepoint <= 0x000D) || + (codepoint >= 0x001C && codepoint <= 0x001F)) { + return true; + } + if (category == UTF8PROC_CATEGORY_ZL || category == UTF8PROC_CATEGORY_ZP) { + return true; + } + return category == UTF8PROC_CATEGORY_ZS && codepoint != 0x00A0 && + codepoint != 0x2007 && codepoint != 0x202F; +} + +uint32_t ngram_token_char_bit(const Codepoint &codepoint) { + auto category = utf8proc_category(codepoint.cp); + if (is_ngram_whitespace(static_cast(codepoint.cp), category)) { + return kNGramTokenCharWhitespace; + } + if (category == UTF8PROC_CATEGORY_ND) { + return kNGramTokenCharDigit; + } + if (category == UTF8PROC_CATEGORY_LU || category == UTF8PROC_CATEGORY_LL || + category == UTF8PROC_CATEGORY_LT || category == UTF8PROC_CATEGORY_LM || + category == UTF8PROC_CATEGORY_LO) { + return kNGramTokenCharLetter; + } + if (category == UTF8PROC_CATEGORY_PC || category == UTF8PROC_CATEGORY_PD || + category == UTF8PROC_CATEGORY_PS || category == UTF8PROC_CATEGORY_PE || + category == UTF8PROC_CATEGORY_PI || category == UTF8PROC_CATEGORY_PF || + category == UTF8PROC_CATEGORY_PO) { + return kNGramTokenCharPunctuation; + } + if (category == UTF8PROC_CATEGORY_SM || category == UTF8PROC_CATEGORY_SC || + category == UTF8PROC_CATEGORY_SK || category == UTF8PROC_CATEGORY_SO) { + return kNGramTokenCharSymbol; + } + return 0; +} + +bool is_ngram_token_char(const Codepoint &codepoint, uint32_t token_char_mask) { + if (!codepoint.valid) { + return false; + } + if (token_char_mask == 0) { + return true; + } + return (ngram_token_char_bit(codepoint) & token_char_mask) != 0; +} + +std::vector> collect_ngram_segments( + const std::string &text, uint32_t token_char_mask) { + std::vector> spans; + spans.reserve(estimate_token_capacity(text.size())); + std::vector codepoints = decode_utf8(text); + std::vector span; + + for (const auto &codepoint : codepoints) { + if (is_ngram_token_char(codepoint, token_char_mask)) { + span.push_back({codepoint.start, codepoint.end}); + continue; + } + if (!span.empty()) { + spans.push_back(std::move(span)); + span.clear(); + } + } + if (!span.empty()) { + spans.push_back(std::move(span)); + } + + return spans; +} + +void emit_ngram_tokens(const std::string &text, + const std::vector &span, + uint32_t ngram_min, uint32_t ngram_max, + uint32_t *position, std::vector *tokens) { + if (span.size() < ngram_min) { + return; + } + + for (size_t start = 0; start < span.size(); ++start) { + size_t remaining = span.size() - start; + size_t upper = std::min(ngram_max, remaining); + for (size_t length = ngram_min; length <= upper; ++length) { + const CodepointSpan &first = span[start]; + const CodepointSpan &last = span[start + length - 1]; + Token token; + token.text = text.substr(first.start, last.end - first.start); + token.offset = first.start; + token.position = (*position)++; + tokens->push_back(std::move(token)); + } + } +} + } // namespace bool StandardTokenizer::init(const ailego::JsonObject &config) { @@ -1097,4 +1280,34 @@ std::vector StandardTokenizer::tokenize(const std::string &text) const { return tokens; } +bool NGramTokenizer::init(const ailego::JsonObject &config) { + if (!parse_positive_uint32_param(config, "ngram_min", kDefaultNGramLength, + &ngram_min_)) { + return false; + } + if (!parse_positive_uint32_param(config, "ngram_max", kDefaultNGramLength, + &ngram_max_)) { + return false; + } + if (ngram_min_ > ngram_max_) { + LOG_ERROR("NGramTokenizer: ngram_min must be <= ngram_max"); + return false; + } + if (!parse_ngram_token_chars(config, &token_char_mask_)) { + return false; + } + return true; +} + +std::vector NGramTokenizer::tokenize(const std::string &text) const { + std::vector tokens; + tokens.reserve(estimate_token_capacity(text.size())); + uint32_t position = 0; + auto spans = collect_ngram_segments(text, token_char_mask_); + for (const auto &span : spans) { + emit_ngram_tokens(text, span, ngram_min_, ngram_max_, &position, &tokens); + } + return tokens; +} + } // namespace zvec::fts diff --git a/src/db/index/column/fts_column/tokenizer/standard_tokenizer.h b/src/db/index/column/fts_column/tokenizer/standard_tokenizer.h index 9c46b3b28..f61ea66a4 100644 --- a/src/db/index/column/fts_column/tokenizer/standard_tokenizer.h +++ b/src/db/index/column/fts_column/tokenizer/standard_tokenizer.h @@ -48,4 +48,36 @@ class StandardTokenizer : public Tokenizer { uint32_t max_token_length_{255}; }; +/*! NGram tokenizer + * Unicode-aware tokenizer that uses the standard tokenizer's UTF-8 decoding, + * Unicode classification and delimiter rules to find base text spans, then + * emits UTF-8 codepoint ngrams from each span. Consecutive CJK text remains in + * the same base span so CJK ngrams can be generated. + */ +class NGramTokenizer : public Tokenizer { + public: + /*! Initialise from JSON config. + * Supported keys: + * "ngram_min" (positive integer, default 2): minimum ngram length. + * "ngram_max" (positive integer, default 2): maximum ngram length. + * "token_chars" (array of strings, default []): character classes included + * in tokens. Supported classes are "letter", "digit", "whitespace", + * "punctuation" and "symbol". Empty array keeps all valid UTF-8 + * characters. + * Returns false when the configuration is invalid. + */ + bool init(const ailego::JsonObject &config) override; + + std::vector tokenize(const std::string &text) const override; + + const char *name() const override { + return "ngram"; + } + + private: + uint32_t ngram_min_{2}; + uint32_t ngram_max_{2}; + uint32_t token_char_mask_{0}; +}; + } // namespace zvec::fts diff --git a/src/db/index/column/fts_column/tokenizer/tokenizer_factory.cc b/src/db/index/column/fts_column/tokenizer/tokenizer_factory.cc index 41927c312..caf75daa8 100644 --- a/src/db/index/column/fts_column/tokenizer/tokenizer_factory.cc +++ b/src/db/index/column/fts_column/tokenizer/tokenizer_factory.cc @@ -82,6 +82,8 @@ TokenizerPtr TokenizerFactory::create_tokenizer( TokenizerPtr tokenizer; if (tokenizer_name.empty() || tokenizer_name == "standard") { tokenizer = std::make_shared(); + } else if (tokenizer_name == "ngram") { + tokenizer = std::make_shared(); } else if (tokenizer_name == "jieba") { tokenizer = std::make_shared(); } else if (tokenizer_name == "whitespace") { diff --git a/src/include/zvec/c_api.h b/src/include/zvec/c_api.h index 169dad656..1b264ebde 100644 --- a/src/include/zvec/c_api.h +++ b/src/include/zvec/c_api.h @@ -1127,7 +1127,7 @@ ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_index_params_set_invert_params( * @brief Set FTS index specific parameters * @param params Index parameters (must be FTS type) * @param tokenizer_name Tokenizer pipeline name (NULL keeps current value). - * Supported values are "standard", "jieba", and "whitespace". + * Supported values are "standard", "ngram", "jieba", and "whitespace". * @param filters Token filter names (NULL keeps current value). Supported * values are "lowercase", "ascii_folding", and "stemmer". * @param extra_params Additional tokenizer/filter parameters (NULL keeps @@ -1136,6 +1136,12 @@ ZVEC_EXPORT zvec_error_code_t ZVEC_CALL zvec_index_params_set_invert_params( * Tokenizers: * standard: * - "max_token_length" (positive integer). + * ngram: + * - "ngram_min" (positive integer, default 2). + * - "ngram_max" (positive integer, default 2). + * - "token_chars" (array of "letter", "digit", "whitespace", + * "punctuation", "symbol"; default [] keeps all valid UTF-8 + * characters). custom_token_chars is not supported. * jieba: * - "jieba_dict_dir" (directory containing jieba.dict.utf8 and * hmm_model.utf8). diff --git a/src/include/zvec/db/index_params.h b/src/include/zvec/db/index_params.h index 31ec5b6c6..fc7d82c40 100644 --- a/src/include/zvec/db/index_params.h +++ b/src/include/zvec/db/index_params.h @@ -717,7 +717,7 @@ class VamanaIndexParams : public VectorIndexParams { /* * FTS (Full-Text Search) index params - * Supported tokenizers: "standard", "jieba", "whitespace". + * Supported tokenizers: "standard", "ngram", "jieba", "whitespace". * Supported filters: "lowercase", "ascii_folding", "stemmer". * * extra_params must be either empty or a JSON object string. Supported keys are @@ -725,6 +725,12 @@ class VamanaIndexParams : public VectorIndexParams { * Tokenizers: * standard: * - "max_token_length" (positive integer). + * ngram: + * - "ngram_min" (positive integer, default 2). + * - "ngram_max" (positive integer, default 2). + * - "token_chars" (array of "letter", "digit", "whitespace", + * "punctuation", "symbol"; default [] keeps all valid UTF-8 + * characters). custom_token_chars is not supported. * jieba: * - "jieba_dict_dir" (directory containing jieba.dict.utf8 and * hmm_model.utf8). diff --git a/tests/db/index/column/fts_column/ngram_tokenizer_test.cc b/tests/db/index/column/fts_column/ngram_tokenizer_test.cc new file mode 100644 index 000000000..1cbfed79c --- /dev/null +++ b/tests/db/index/column/fts_column/ngram_tokenizer_test.cc @@ -0,0 +1,187 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include "db/index/column/fts_column/fts_types.h" +#include "db/index/column/fts_column/tokenizer/tokenizer_factory.h" + +namespace { + +using zvec::fts::FtsIndexParams; +using zvec::fts::Token; +using zvec::fts::TokenizerFactory; +using zvec::fts::TokenizerPipelinePtr; + +std::vector token_texts(const std::vector &tokens) { + std::vector texts; + texts.reserve(tokens.size()); + for (const auto &token : tokens) { + texts.push_back(token.text); + } + return texts; +} + +TokenizerPipelinePtr make_pipeline(const std::string &extra_params = "") { + FtsIndexParams params; + params.tokenizer_name = "ngram"; + params.filters.clear(); + params.extra_params = extra_params; + return TokenizerFactory::create(params); +} + +class NGramTokenizerTest : public ::testing::Test { + protected: + void SetUp() override { + pipeline_ = make_pipeline(); + ASSERT_NE(pipeline_, nullptr); + } + + std::vector tokenize(const std::string &text) { + return pipeline_->process(text); + } + + TokenizerPipelinePtr pipeline_; +}; + +TEST_F(NGramTokenizerTest, DefaultBigramAscii) { + auto tokens = tokenize("hello"); + std::vector expected = {"he", "el", "ll", "lo"}; + EXPECT_EQ(token_texts(tokens), expected); +} + +TEST_F(NGramTokenizerTest, CustomRange) { + auto pipeline = make_pipeline(R"({"ngram_min":2,"ngram_max":3})"); + ASSERT_NE(pipeline, nullptr); + + auto tokens = pipeline->process("hello"); + std::vector expected = {"he", "hel", "el", "ell", + "ll", "llo", "lo"}; + EXPECT_EQ(token_texts(tokens), expected); +} + +TEST_F(NGramTokenizerTest, ChineseBigram) { + auto tokens = tokenize("\xE4\xB8\xAD\xE6\x96\x87\xE5\x88\x86\xE8\xAF\x8D"); + std::vector expected = {"\xE4\xB8\xAD\xE6\x96\x87", + "\xE6\x96\x87\xE5\x88\x86", + "\xE5\x88\x86\xE8\xAF\x8D"}; + EXPECT_EQ(token_texts(tokens), expected); +} + +TEST_F(NGramTokenizerTest, DefaultTokenCharsKeepsAllValidCharacters) { + auto tokens = tokenize( + "foobar\xE6\x9C\xAA\xE8\xB7\x9F\xE8\xB8\xAA" + "\xE6\x96\x87\xE4\xBB\xB6"); + std::vector expected = {"fo", + "oo", + "ob", + "ba", + "ar", + "r\xE6\x9C\xAA", + "\xE6\x9C\xAA\xE8\xB7\x9F", + "\xE8\xB7\x9F\xE8\xB8\xAA", + "\xE8\xB8\xAA\xE6\x96\x87", + "\xE6\x96\x87\xE4\xBB\xB6"}; + EXPECT_EQ(token_texts(tokens), expected); +} + +TEST_F(NGramTokenizerTest, DistinguishesNullCodepointFromMalformedUtf8) { + auto null_tokens = tokenize(std::string("a\0b", 3)); + std::vector null_expected = {std::string("a\0", 2), + std::string("\0b", 2)}; + EXPECT_EQ(token_texts(null_tokens), null_expected); + + std::string malformed = "ab"; + malformed.push_back(static_cast(0xFF)); + malformed += "cd"; + auto malformed_tokens = tokenize(malformed); + std::vector malformed_expected = {"ab", "cd"}; + EXPECT_EQ(token_texts(malformed_tokens), malformed_expected); +} + +TEST_F(NGramTokenizerTest, LetterAndDigitTokenCharsSplitOnPunctuation) { + auto pipeline = make_pipeline( + R"({"ngram_min":2,"ngram_max":2,"token_chars":["letter","digit"]})"); + ASSERT_NE(pipeline, nullptr); + + auto tokens = pipeline->process( + "ab cd,\xE4\xB8\xAD\xE6\x96\x87!" + "de"); + std::vector expected = {"ab", "cd", "\xE4\xB8\xAD\xE6\x96\x87", + "de"}; + EXPECT_EQ(token_texts(tokens), expected); +} + +TEST_F(NGramTokenizerTest, WhitespacePunctuationAndSymbolTokenChars) { + auto pipeline = make_pipeline( + R"({"ngram_min":2,"ngram_max":2,"token_chars":["whitespace","punctuation","symbol"]})"); + ASSERT_NE(pipeline, nullptr); + + auto tokens = pipeline->process("a !$b"); + std::vector expected = {" !", "!$"}; + EXPECT_EQ(token_texts(tokens), expected); +} + +TEST_F(NGramTokenizerTest, TokenCharClassesMatchElasticsearchCategories) { + auto whitespace_pipeline = make_pipeline( + R"({"ngram_min":1,"ngram_max":1,"token_chars":["whitespace"]})"); + ASSERT_NE(whitespace_pipeline, nullptr); + auto whitespace_tokens = + whitespace_pipeline->process("\t \n\xE2\x80\xA8\xC2\xA0"); + std::vector whitespace_expected = {"\t", " ", "\n", + "\xE2\x80\xA8"}; + EXPECT_EQ(token_texts(whitespace_tokens), whitespace_expected); + + auto symbol_pipeline = make_pipeline( + R"({"ngram_min":1,"ngram_max":1,"token_chars":["symbol"]})"); + ASSERT_NE(symbol_pipeline, nullptr); + std::string symbol_text = "$\xCC\x81"; + symbol_text.push_back('\x01'); + symbol_text += "\xE2\x88\x9A"; + auto symbol_tokens = symbol_pipeline->process(symbol_text); + std::vector symbol_expected = {"$", "\xE2\x88\x9A"}; + EXPECT_EQ(token_texts(symbol_tokens), symbol_expected); +} + +TEST_F(NGramTokenizerTest, Utf8OffsetsUseOriginalByteOffsets) { + auto tokens = tokenize("\xE4\xB8\xAD\xE6\x96\x87\xE5\x88\x86"); + ASSERT_EQ(tokens.size(), 2u); + EXPECT_EQ(tokens[0].text, "\xE4\xB8\xAD\xE6\x96\x87"); + EXPECT_EQ(tokens[0].offset, 0u); + EXPECT_EQ(tokens[1].text, "\xE6\x96\x87\xE5\x88\x86"); + EXPECT_EQ(tokens[1].offset, 3u); +} + +TEST_F(NGramTokenizerTest, PositionsAreConsecutiveAcrossSegments) { + auto pipeline = make_pipeline(R"({"token_chars":["letter"]})"); + ASSERT_NE(pipeline, nullptr); + auto tokens = pipeline->process("abcd \xE4\xB8\xAD\xE6\x96\x87\xE5\x88\x86"); + ASSERT_EQ(tokens.size(), 5u); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(tokens[i].position, static_cast(i)); + } +} + +TEST(NGramTokenizerConfigTest, InvalidConfigFailsPipelineCreation) { + EXPECT_EQ(make_pipeline(R"({"ngram_min":"2"})"), nullptr); + EXPECT_EQ(make_pipeline(R"({"ngram_min":0})"), nullptr); + EXPECT_EQ(make_pipeline(R"({"ngram_max":0})"), nullptr); + EXPECT_EQ(make_pipeline(R"({"ngram_min":3,"ngram_max":2})"), nullptr); + EXPECT_EQ(make_pipeline(R"({"token_chars":"letter"})"), nullptr); + EXPECT_EQ(make_pipeline(R"({"token_chars":[1]})"), nullptr); + EXPECT_EQ(make_pipeline(R"({"token_chars":["custom"]})"), nullptr); +} + +} // namespace