From 9a8eadda4e3837b2c2986aeb522189a98b3ca674 Mon Sep 17 00:00:00 2001 From: yhirose Date: Tue, 9 Jun 2026 23:16:02 -0400 Subject: [PATCH 1/2] Simplify decode() and validate UTF-8 continuation bytes Remove the internal for_each() helpers and let decode() call decode_codepoint() directly. This drops the duplicated scan that for_each() and decode_codepoint() each performed, simplifying the code and speeding up decoding (~20% for UTF-16, ~30% for UTF-8). The helpers were used nowhere outside decode(). Based on #15 by Sam Hocevar. Also validate continuation bytes in the UTF-8 decode_codepoint(). Without the for_each() grouping, an invalid lead byte would otherwise consume the following bytes unconditionally (they were only masked, never checked), swallowing trailing valid characters -- e.g. "\xE0AB" decoded to a single bogus code point instead of "AB". Rejecting bytes that are not 0x80-0xBF makes decode() resync byte-by-byte and recover the following text, matching the behaviour of ICU, Python, Rust and browsers (minus U+FFFD emission). The UTF-16 path already validated surrogate pairs and is unchanged in behaviour. --- unicodelib_encodings.h | 63 ++++++++++++------------------------------ 1 file changed, 18 insertions(+), 45 deletions(-) diff --git a/unicodelib_encodings.h b/unicodelib_encodings.h index 8683b67..b3a55b7 100644 --- a/unicodelib_encodings.h +++ b/unicodelib_encodings.h @@ -161,14 +161,14 @@ inline bool decode_codepoint(const char *s8, size_t l, size_t &bytes, cp = b; return true; } else if ((b & 0xE0) == 0xC0) { - if (l >= 2) { + if (l >= 2 && (s8[1] & 0xC0) == 0x80) { bytes = 2; cp = ((static_cast(s8[0] & 0x1F)) << 6) | (static_cast(s8[1] & 0x3F)); return true; } } else if ((b & 0xF0) == 0xE0) { - if (l >= 3) { + if (l >= 3 && (s8[1] & 0xC0) == 0x80 && (s8[2] & 0xC0) == 0x80) { bytes = 3; cp = ((static_cast(s8[0] & 0x0F)) << 12) | ((static_cast(s8[1] & 0x3F)) << 6) | @@ -176,7 +176,8 @@ inline bool decode_codepoint(const char *s8, size_t l, size_t &bytes, return true; } } else if ((b & 0xF8) == 0xF0) { - if (l >= 4) { + if (l >= 4 && (s8[1] & 0xC0) == 0x80 && (s8[2] & 0xC0) == 0x80 && + (s8[3] & 0xC0) == 0x80) { bytes = 4; cp = ((static_cast(s8[0] & 0x07)) << 18) | ((static_cast(s8[1] & 0x3F)) << 12) | @@ -197,31 +198,17 @@ inline size_t decode_codepoint(const char *s8, size_t l, char32_t &out) { return 0; } -template -inline void for_each(const char *s8, size_t l, T callback) { - size_t id = 0; - size_t i = 0; - while (i < l) { - auto beg = i++; - while (i < l && (s8[i] & 0xc0) == 0x80) { - i++; +inline void decode(const char *s8, size_t l, std::u32string &out) { + for (size_t i = 0, bytes; i < l; i += bytes) { + char32_t cp; + if (decode_codepoint(&s8[i], l - i, bytes, cp)) { + out += cp; + } else { + bytes = 1; } - callback(s8, l, beg, i, id++); } } -inline void decode(const char *s8, size_t l, std::u32string &out) { - for_each( - s8, l, - [&](const char *s, size_t /*l*/, size_t beg, size_t end, size_t /*i*/) { - size_t bytes; - char32_t cp; - if (decode_codepoint(&s[beg], (end - beg), bytes, cp)) { - out += cp; - } - }); -} - } // namespace utf8 //----------------------------------------------------------------------------- @@ -331,31 +318,17 @@ inline size_t decode_codepoint(const char16_t *s16, size_t l, char32_t &out) { return 0; } -template -inline void for_each(const char16_t *s16, size_t l, T callback) { - size_t id = 0; - size_t i = 0; - while (i < l) { - auto beg = i++; - if (is_surrogate_pair(&s16[beg], l - beg)) { - i++; +inline void decode(const char16_t *s16, size_t l, std::u32string &out) { + for (size_t i = 0, length; i < l; i += length) { + char32_t cp; + if (decode_codepoint(&s16[i], l - i, length, cp)) { + out += cp; + } else { + length = 1; } - callback(s16, l, beg, i, id++); } } -inline void decode(const char16_t *s16, size_t l, std::u32string &out) { - for_each(s16, l, - [&](const char16_t *s, size_t /*l*/, size_t beg, size_t end, - size_t /*i*/) { - size_t length; - char32_t cp; - if (decode_codepoint(&s[beg], (end - beg), length, cp)) { - out += cp; - } - }); -} - } // namespace utf16 //----------------------------------------------------------------------------- From 900291ebeac8bfd6e8fa262f5319a9a6e27e0e3c Mon Sep 17 00:00:00 2001 From: yhirose Date: Tue, 9 Jun 2026 23:23:06 -0400 Subject: [PATCH 2/2] Add tests for UTF-8 continuation-byte validation and resync Cover that an invalid UTF-8 lead byte resyncs at the next byte instead of swallowing the following valid characters, that decode_codepoint() reports failure on bad continuation bytes, and the matching lone-surrogate resync for UTF-16. These fail against the un-validated decode and pass with it. --- test/test.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/test.cpp b/test/test.cpp index cb0c63a..803d6c1 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -529,6 +529,25 @@ TEST_CASE("decode 3", "[utf8]") { REQUIRE(utf8::decode(u8text) == u32text); } +TEST_CASE("decode invalid utf8 resync", "[utf8]") { + // An invalid lead byte must resync at the next byte and must not swallow the + // following valid characters. + REQUIRE(utf8::decode("\xE0" "AB") == U"AB"); // truncated 3-byte lead + REQUIRE(utf8::decode("\xF0" "Hello") == U"Hello"); // truncated 4-byte lead + REQUIRE(utf8::decode("\xE1\x80" "A") == U"A"); // bad 3rd byte + REQUIRE(utf8::decode("a\x80" "b") == U"ab"); // lone continuation byte + REQUIRE(utf8::decode("\xFF\xFF" "x") == U"x"); // never-valid lead bytes +} + +TEST_CASE("decode_codepoint rejects bad continuation bytes", "[utf8]") { + char32_t cp; + // A continuation byte that is not 0x80-0xBF makes decode fail (0 bytes). + REQUIRE(utf8::decode_codepoint("\xE0" "AB", 3, cp) == 0); + REQUIRE(utf8::decode_codepoint("\xF0\x41\x42\x43", 4, cp) == 0); + // A well-formed sequence is still accepted and reports its length. + REQUIRE(utf8::decode_codepoint(u8"あ", 3, cp) == 3); +} + } // namespace test_utf8 //----------------------------------------------------------------------------- @@ -613,6 +632,12 @@ TEST_CASE("utf16 decode 3", "[utf16]") { REQUIRE(utf16::decode(u16text) == u32text); } +TEST_CASE("utf16 decode invalid resync", "[utf16]") { + // A lone high surrogate must resync and must not swallow the next unit. + const char16_t lone_high[] = {0xD800, u'A'}; + REQUIRE(utf16::decode(std::u16string_view(lone_high, 2)) == U"A"); +} + } // namespace test_utf16 //-----------------------------------------------------------------------------