From 04858f8f0d205790b204703d3ca277d2676fd2b0 Mon Sep 17 00:00:00 2001 From: "logan.riggs@gmail.com" Date: Tue, 2 Jun 2026 21:02:50 +0000 Subject: [PATCH 1/2] DX-116527 Update CHR to work with unicode --- cpp/src/gandiva/precompiled/string_ops.cc | 62 +++++++--- .../gandiva/precompiled/string_ops_test.cc | 109 +++++++++++------- 2 files changed, 111 insertions(+), 60 deletions(-) diff --git a/cpp/src/gandiva/precompiled/string_ops.cc b/cpp/src/gandiva/precompiled/string_ops.cc index 035d3c8c62e1..61187ec471bc 100644 --- a/cpp/src/gandiva/precompiled/string_ops.cc +++ b/cpp/src/gandiva/precompiled/string_ops.cc @@ -1387,29 +1387,27 @@ gdv_int32 ascii_utf8(const char* data, gdv_int32 data_len) { return static_cast(static_cast(data[0])); } -// Returns the ASCII character having the binary equivalent to A. -// If A is larger than 256 the result is equivalent to chr(A % 256). +// Returns the UTF-8 encoding of the Unicode code point A. +// Raises an error if A is not a valid code point, i.e. it is negative, greater +// than 0x10FFFF, or falls within the UTF-16 surrogate range 0xD800-0xDFFF. FORCE_INLINE -const char* chr_int32(gdv_int64 context, gdv_int32 in, gdv_int32* out_len) { - in = in % 256; - *out_len = 1; - - char* ret = reinterpret_cast(gdv_fn_context_arena_malloc(context, *out_len)); - if (ret == nullptr) { - gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); +const char* chr_int64(gdv_int64 context, gdv_int64 in, gdv_int32* out_len) { + if (in < 0 || in > 0x10FFFF || (in >= 0xD800 && in <= 0xDFFF)) { + gdv_fn_context_set_error_msg( + context, "Input is not a valid Unicode code point in the range 0 to 0x10FFFF"); *out_len = 0; return ""; } - ret[0] = char(in); - return ret; -} -// Returns the ASCII character having the binary equivalent to A. -// If A is larger than 256 the result is equivalent to chr(A % 256). -FORCE_INLINE -const char* chr_int64(gdv_int64 context, gdv_int64 in, gdv_int32* out_len) { - in = in % 256; - *out_len = 1; + if (in <= 0x7F) { + *out_len = 1; + } else if (in <= 0x7FF) { + *out_len = 2; + } else if (in <= 0xFFFF) { + *out_len = 3; + } else { + *out_len = 4; + } char* ret = reinterpret_cast(gdv_fn_context_arena_malloc(context, *out_len)); if (ret == nullptr) { @@ -1417,10 +1415,36 @@ const char* chr_int64(gdv_int64 context, gdv_int64 in, gdv_int32* out_len) { *out_len = 0; return ""; } - ret[0] = char(in); + + switch (*out_len) { + case 1: + ret[0] = static_cast(in); + break; + case 2: + ret[0] = static_cast(0xC0 | (in >> 6)); + ret[1] = static_cast(0x80 | (in & 0x3F)); + break; + case 3: + ret[0] = static_cast(0xE0 | (in >> 12)); + ret[1] = static_cast(0x80 | ((in >> 6) & 0x3F)); + ret[2] = static_cast(0x80 | (in & 0x3F)); + break; + case 4: + ret[0] = static_cast(0xF0 | (in >> 18)); + ret[1] = static_cast(0x80 | ((in >> 12) & 0x3F)); + ret[2] = static_cast(0x80 | ((in >> 6) & 0x3F)); + ret[3] = static_cast(0x80 | (in & 0x3F)); + break; + } return ret; } +// Returns the UTF-8 encoding of the Unicode code point A. See chr_int64. +FORCE_INLINE +const char* chr_int32(gdv_int64 context, gdv_int32 in, gdv_int32* out_len) { + return chr_int64(context, in, out_len); +} + FORCE_INLINE const char* convert_fromUTF8_binary(gdv_int64 context, const char* bin_in, gdv_int32 len, gdv_int32* out_len) { diff --git a/cpp/src/gandiva/precompiled/string_ops_test.cc b/cpp/src/gandiva/precompiled/string_ops_test.cc index ea661585ecb5..2e64a0dd7469 100644 --- a/cpp/src/gandiva/precompiled/string_ops_test.cc +++ b/cpp/src/gandiva/precompiled/string_ops_test.cc @@ -56,11 +56,12 @@ TEST(TestStringOps, TestAscii) { } TEST(TestStringOps, TestChrBigInt) { - // CHR + // CHR returns the UTF-8 encoding of the given Unicode code point. gandiva::ExecutionContext ctx; uint64_t ctx_ptr = reinterpret_cast(&ctx); int32_t out_len = 0; + // 1-byte ASCII code points. auto out = chr_int32(ctx_ptr, 88, &out_len); EXPECT_EQ(std::string(out, out_len), "X"); @@ -70,62 +71,88 @@ TEST(TestStringOps, TestChrBigInt) { out = chr_int32(ctx_ptr, 49, &out_len); EXPECT_EQ(std::string(out, out_len), "1"); - out = chr_int64(ctx_ptr, 84, &out_len); - EXPECT_EQ(std::string(out, out_len), "T"); + out = chr_int32(ctx_ptr, 33, &out_len); + EXPECT_EQ(std::string(out, out_len), "!"); - out = chr_int32(ctx_ptr, 340, &out_len); - EXPECT_EQ(std::string(out, out_len), "T"); + out = chr_int64(ctx_ptr, 0, &out_len); + EXPECT_EQ(std::string(out, out_len), std::string("\0", 1)); - out = chr_int64(ctx_ptr, 256, &out_len); - EXPECT_EQ(std::strcmp(out, "\0"), 0); + // BACKSPACE + out = chr_int64(ctx_ptr, 8, &out_len); + EXPECT_EQ(std::string(out, out_len), "\b"); - out = chr_int32(ctx_ptr, 33, &out_len); - EXPECT_EQ(std::string(out, out_len), "!"); + // ESCAPE (ESC) + out = chr_int64(ctx_ptr, 27, &out_len); + EXPECT_EQ(std::string(out, out_len), "\x1B"); - out = chr_int64(ctx_ptr, 46, &out_len); - EXPECT_EQ(std::string(out, out_len), "."); + // Highest 1-byte code point (U+007F, DELETE). + out = chr_int32(ctx_ptr, 0x7F, &out_len); + EXPECT_EQ(std::string(out, out_len), "\x7F"); - out = chr_int32(ctx_ptr, 63, &out_len); - EXPECT_EQ(std::string(out, out_len), "?"); + // 2-byte code points. + // Lowest 2-byte code point (U+0080). + out = chr_int32(ctx_ptr, 0x80, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xC2\x80"); - out = chr_int64(ctx_ptr, 0, &out_len); - EXPECT_EQ(std::strcmp(out, "\0"), 0); + // í (U+00ED) + out = chr_int32(ctx_ptr, 237, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xC3\xAD"); - out = chr_int32(ctx_ptr, -158, &out_len); - EXPECT_EQ(std::string(out, out_len), "b"); + // Highest 2-byte code point (U+07FF). + out = chr_int64(ctx_ptr, 0x7FF, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xDF\xBF"); - out = chr_int64(ctx_ptr, -5, &out_len); - EXPECT_EQ(std::string(out, out_len), "\xFB"); + // 3-byte code points. + // Lowest 3-byte code point (U+0800). + out = chr_int32(ctx_ptr, 0x800, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xE0\xA0\x80"); - out = chr_int32(ctx_ptr, -340, &out_len); - EXPECT_EQ(std::string(out, out_len), "\xAC"); + // € (U+20AC) + out = chr_int32(ctx_ptr, 8364, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xE2\x82\xAC"); - out = chr_int64(ctx_ptr, -66, &out_len); - EXPECT_EQ(std::string(out, out_len), "\xBE"); + // 日 (U+65E5) + out = chr_int64(ctx_ptr, 26085, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xE6\x97\xA5"); - //€ - out = chr_int32(ctx_ptr, 128, &out_len); - EXPECT_EQ(std::string(out, out_len), "\x80"); + // Highest 3-byte code point (U+FFFF). + out = chr_int32(ctx_ptr, 0xFFFF, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xEF\xBF\xBF"); - //œ - out = chr_int64(ctx_ptr, 156, &out_len); - EXPECT_EQ(std::string(out, out_len), "\x9C"); + // 4-byte code points. + // Lowest 4-byte code point (U+10000). + out = chr_int64(ctx_ptr, 0x10000, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xF0\x90\x80\x80"); - //ÿ - out = chr_int32(ctx_ptr, 255, &out_len); - EXPECT_EQ(std::string(out, out_len), "\xFF"); + // 😀 (U+1F600) + out = chr_int64(ctx_ptr, 0x1F600, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xF0\x9F\x98\x80"); - // BACKSPACE - out = chr_int64(ctx_ptr, 8, &out_len); - EXPECT_EQ(std::string(out, out_len), "\b"); + // Highest valid code point (U+10FFFF). + out = chr_int64(ctx_ptr, 0x10FFFF, &out_len); + EXPECT_EQ(std::string(out, out_len), "\xF4\x8F\xBF\xBF"); - // DEVICE CONTROL 3 (DC3) - out = chr_int32(ctx_ptr, 19, &out_len); - EXPECT_EQ(std::string(out, out_len), "\x13"); + EXPECT_FALSE(ctx.has_error()); - // ESCAPE (ESC) - out = chr_int64(ctx_ptr, 27, &out_len); - EXPECT_EQ(std::string(out, out_len), "\x1B"); + // Invalid code points raise an error. + chr_int64(ctx_ptr, -1, &out_len); + EXPECT_EQ(out_len, 0); + EXPECT_TRUE(ctx.get_error().find("not a valid Unicode code point") != std::string::npos) + << ctx.get_error(); + ctx.Reset(); + + chr_int64(ctx_ptr, 0x110000, &out_len); + EXPECT_EQ(out_len, 0); + EXPECT_TRUE(ctx.get_error().find("not a valid Unicode code point") != std::string::npos) + << ctx.get_error(); + ctx.Reset(); + + // UTF-16 surrogate range is not a valid code point. + chr_int32(ctx_ptr, 0xD800, &out_len); + EXPECT_EQ(out_len, 0); + EXPECT_TRUE(ctx.get_error().find("not a valid Unicode code point") != std::string::npos) + << ctx.get_error(); + ctx.Reset(); } TEST(TestStringOps, TestBeginsEnds) { From 445bf8a6fa65c9fb128c4d30ed09e1c0923d23b1 Mon Sep 17 00:00:00 2001 From: "logan.riggs@gmail.com" Date: Tue, 2 Jun 2026 22:02:59 +0000 Subject: [PATCH 2/2] Update error message for readability. --- cpp/src/gandiva/precompiled/string_ops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/gandiva/precompiled/string_ops.cc b/cpp/src/gandiva/precompiled/string_ops.cc index 61187ec471bc..cc3e82a7e6e5 100644 --- a/cpp/src/gandiva/precompiled/string_ops.cc +++ b/cpp/src/gandiva/precompiled/string_ops.cc @@ -1394,7 +1394,7 @@ FORCE_INLINE const char* chr_int64(gdv_int64 context, gdv_int64 in, gdv_int32* out_len) { if (in < 0 || in > 0x10FFFF || (in >= 0xD800 && in <= 0xDFFF)) { gdv_fn_context_set_error_msg( - context, "Input is not a valid Unicode code point in the range 0 to 0x10FFFF"); + context, "Input is not a valid Unicode code point."); *out_len = 0; return ""; }