diff --git a/test/test.cpp b/test/test.cpp index cb0c63a..803d6c1 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -529,6 +529,25 @@ TEST_CASE("decode 3", "[utf8]") { REQUIRE(utf8::decode(u8text) == u32text); } +TEST_CASE("decode invalid utf8 resync", "[utf8]") { + // An invalid lead byte must resync at the next byte and must not swallow the + // following valid characters. + REQUIRE(utf8::decode("\xE0" "AB") == U"AB"); // truncated 3-byte lead + REQUIRE(utf8::decode("\xF0" "Hello") == U"Hello"); // truncated 4-byte lead + REQUIRE(utf8::decode("\xE1\x80" "A") == U"A"); // bad 3rd byte + REQUIRE(utf8::decode("a\x80" "b") == U"ab"); // lone continuation byte + REQUIRE(utf8::decode("\xFF\xFF" "x") == U"x"); // never-valid lead bytes +} + +TEST_CASE("decode_codepoint rejects bad continuation bytes", "[utf8]") { + char32_t cp; + // A continuation byte that is not 0x80-0xBF makes decode fail (0 bytes). + REQUIRE(utf8::decode_codepoint("\xE0" "AB", 3, cp) == 0); + REQUIRE(utf8::decode_codepoint("\xF0\x41\x42\x43", 4, cp) == 0); + // A well-formed sequence is still accepted and reports its length. + REQUIRE(utf8::decode_codepoint(u8"あ", 3, cp) == 3); +} + } // namespace test_utf8 //----------------------------------------------------------------------------- @@ -613,6 +632,12 @@ TEST_CASE("utf16 decode 3", "[utf16]") { REQUIRE(utf16::decode(u16text) == u32text); } +TEST_CASE("utf16 decode invalid resync", "[utf16]") { + // A lone high surrogate must resync and must not swallow the next unit. + const char16_t lone_high[] = {0xD800, u'A'}; + REQUIRE(utf16::decode(std::u16string_view(lone_high, 2)) == U"A"); +} + } // namespace test_utf16 //----------------------------------------------------------------------------- diff --git a/unicodelib_encodings.h b/unicodelib_encodings.h index 8683b67..b3a55b7 100644 --- a/unicodelib_encodings.h +++ b/unicodelib_encodings.h @@ -161,14 +161,14 @@ inline bool decode_codepoint(const char *s8, size_t l, size_t &bytes, cp = b; return true; } else if ((b & 0xE0) == 0xC0) { - if (l >= 2) { + if (l >= 2 && (s8[1] & 0xC0) == 0x80) { bytes = 2; cp = ((static_cast(s8[0] & 0x1F)) << 6) | (static_cast(s8[1] & 0x3F)); return true; } } else if ((b & 0xF0) == 0xE0) { - if (l >= 3) { + if (l >= 3 && (s8[1] & 0xC0) == 0x80 && (s8[2] & 0xC0) == 0x80) { bytes = 3; cp = ((static_cast(s8[0] & 0x0F)) << 12) | ((static_cast(s8[1] & 0x3F)) << 6) | @@ -176,7 +176,8 @@ inline bool decode_codepoint(const char *s8, size_t l, size_t &bytes, return true; } } else if ((b & 0xF8) == 0xF0) { - if (l >= 4) { + if (l >= 4 && (s8[1] & 0xC0) == 0x80 && (s8[2] & 0xC0) == 0x80 && + (s8[3] & 0xC0) == 0x80) { bytes = 4; cp = ((static_cast(s8[0] & 0x07)) << 18) | ((static_cast(s8[1] & 0x3F)) << 12) | @@ -197,31 +198,17 @@ inline size_t decode_codepoint(const char *s8, size_t l, char32_t &out) { return 0; } -template -inline void for_each(const char *s8, size_t l, T callback) { - size_t id = 0; - size_t i = 0; - while (i < l) { - auto beg = i++; - while (i < l && (s8[i] & 0xc0) == 0x80) { - i++; +inline void decode(const char *s8, size_t l, std::u32string &out) { + for (size_t i = 0, bytes; i < l; i += bytes) { + char32_t cp; + if (decode_codepoint(&s8[i], l - i, bytes, cp)) { + out += cp; + } else { + bytes = 1; } - callback(s8, l, beg, i, id++); } } -inline void decode(const char *s8, size_t l, std::u32string &out) { - for_each( - s8, l, - [&](const char *s, size_t /*l*/, size_t beg, size_t end, size_t /*i*/) { - size_t bytes; - char32_t cp; - if (decode_codepoint(&s[beg], (end - beg), bytes, cp)) { - out += cp; - } - }); -} - } // namespace utf8 //----------------------------------------------------------------------------- @@ -331,31 +318,17 @@ inline size_t decode_codepoint(const char16_t *s16, size_t l, char32_t &out) { return 0; } -template -inline void for_each(const char16_t *s16, size_t l, T callback) { - size_t id = 0; - size_t i = 0; - while (i < l) { - auto beg = i++; - if (is_surrogate_pair(&s16[beg], l - beg)) { - i++; +inline void decode(const char16_t *s16, size_t l, std::u32string &out) { + for (size_t i = 0, length; i < l; i += length) { + char32_t cp; + if (decode_codepoint(&s16[i], l - i, length, cp)) { + out += cp; + } else { + length = 1; } - callback(s16, l, beg, i, id++); } } -inline void decode(const char16_t *s16, size_t l, std::u32string &out) { - for_each(s16, l, - [&](const char16_t *s, size_t /*l*/, size_t beg, size_t end, - size_t /*i*/) { - size_t length; - char32_t cp; - if (decode_codepoint(&s[beg], (end - beg), length, cp)) { - out += cp; - } - }); -} - } // namespace utf16 //-----------------------------------------------------------------------------