Simplify decode() and validate UTF-8 continuation bytes

test/test.cpp

@@ -529,6 +529,25 @@ TEST_CASE("decode 3", "[utf8]") {
   REQUIRE(utf8::decode(u8text) == u32text);
 }
 
+TEST_CASE("decode invalid utf8 resync", "[utf8]") {
+  // An invalid lead byte must resync at the next byte and must not swallow the
+  // following valid characters.
+  REQUIRE(utf8::decode("\xE0" "AB") == U"AB");        // truncated 3-byte lead
+  REQUIRE(utf8::decode("\xF0" "Hello") == U"Hello");  // truncated 4-byte lead
+  REQUIRE(utf8::decode("\xE1\x80" "A") == U"A");       // bad 3rd byte
+  REQUIRE(utf8::decode("a\x80" "b") == U"ab");         // lone continuation byte
+  REQUIRE(utf8::decode("\xFF\xFF" "x") == U"x");        // never-valid lead bytes
+}
+
+TEST_CASE("decode_codepoint rejects bad continuation bytes", "[utf8]") {
+  char32_t cp;
+  // A continuation byte that is not 0x80-0xBF makes decode fail (0 bytes).
+  REQUIRE(utf8::decode_codepoint("\xE0" "AB", 3, cp) == 0);
+  REQUIRE(utf8::decode_codepoint("\xF0\x41\x42\x43", 4, cp) == 0);
+  // A well-formed sequence is still accepted and reports its length.
+  REQUIRE(utf8::decode_codepoint(u8"あ", 3, cp) == 3);
+}
+
 }  // namespace test_utf8
 
 //-----------------------------------------------------------------------------
@@ -613,6 +632,12 @@ TEST_CASE("utf16 decode 3", "[utf16]") {
   REQUIRE(utf16::decode(u16text) == u32text);
 }
 
+TEST_CASE("utf16 decode invalid resync", "[utf16]") {
+  // A lone high surrogate must resync and must not swallow the next unit.
+  const char16_t lone_high[] = {0xD800, u'A'};
+  REQUIRE(utf16::decode(std::u16string_view(lone_high, 2)) == U"A");
+}
+
 }  // namespace test_utf16
 
 //-----------------------------------------------------------------------------

unicodelib_encodings.h

@@ -161,22 +161,23 @@ inline bool decode_codepoint(const char *s8, size_t l, size_t &bytes,
       cp = b;
       return true;
     } else if ((b & 0xE0) == 0xC0) {
-      if (l >= 2) {
+      if (l >= 2 && (s8[1] & 0xC0) == 0x80) {
         bytes = 2;
         cp = ((static_cast<char32_t>(s8[0] & 0x1F)) << 6) |
              (static_cast<char32_t>(s8[1] & 0x3F));
         return true;
       }
     } else if ((b & 0xF0) == 0xE0) {
-      if (l >= 3) {
+      if (l >= 3 && (s8[1] & 0xC0) == 0x80 && (s8[2] & 0xC0) == 0x80) {
         bytes = 3;
         cp = ((static_cast<char32_t>(s8[0] & 0x0F)) << 12) |
              ((static_cast<char32_t>(s8[1] & 0x3F)) << 6) |
              (static_cast<char32_t>(s8[2] & 0x3F));
         return true;
       }
     } else if ((b & 0xF8) == 0xF0) {
-      if (l >= 4) {
+      if (l >= 4 && (s8[1] & 0xC0) == 0x80 && (s8[2] & 0xC0) == 0x80 &&
+          (s8[3] & 0xC0) == 0x80) {
         bytes = 4;
         cp = ((static_cast<char32_t>(s8[0] & 0x07)) << 18) |
              ((static_cast<char32_t>(s8[1] & 0x3F)) << 12) |
@@ -197,31 +198,17 @@ inline size_t decode_codepoint(const char *s8, size_t l, char32_t &out) {
   return 0;
 }
 
-template <typename T>
-inline void for_each(const char *s8, size_t l, T callback) {
-  size_t id = 0;
-  size_t i = 0;
-  while (i < l) {
-    auto beg = i++;
-    while (i < l && (s8[i] & 0xc0) == 0x80) {
-      i++;
+inline void decode(const char *s8, size_t l, std::u32string &out) {
+  for (size_t i = 0, bytes; i < l; i += bytes) {
+    char32_t cp;
+    if (decode_codepoint(&s8[i], l - i, bytes, cp)) {
+      out += cp;
+    } else {
+      bytes = 1;
     }
-    callback(s8, l, beg, i, id++);
   }
 }
 
-inline void decode(const char *s8, size_t l, std::u32string &out) {
-  for_each(
-      s8, l,
-      [&](const char *s, size_t /*l*/, size_t beg, size_t end, size_t /*i*/) {
-        size_t bytes;
-        char32_t cp;
-        if (decode_codepoint(&s[beg], (end - beg), bytes, cp)) {
-          out += cp;
-        }
-      });
-}
-
 }  // namespace utf8
 
 //-----------------------------------------------------------------------------
@@ -331,31 +318,17 @@ inline size_t decode_codepoint(const char16_t *s16, size_t l, char32_t &out) {
   return 0;
 }
 
-template <typename T>
-inline void for_each(const char16_t *s16, size_t l, T callback) {
-  size_t id = 0;
-  size_t i = 0;
-  while (i < l) {
-    auto beg = i++;
-    if (is_surrogate_pair(&s16[beg], l - beg)) {
-      i++;
+inline void decode(const char16_t *s16, size_t l, std::u32string &out) {
+  for (size_t i = 0, length; i < l; i += length) {
+    char32_t cp;
+    if (decode_codepoint(&s16[i], l - i, length, cp)) {
+      out += cp;
+    } else {
+      length = 1;
     }
-    callback(s16, l, beg, i, id++);
   }
 }
 
-inline void decode(const char16_t *s16, size_t l, std::u32string &out) {
-  for_each(s16, l,
-           [&](const char16_t *s, size_t /*l*/, size_t beg, size_t end,
-               size_t /*i*/) {
-             size_t length;
-             char32_t cp;
-             if (decode_codepoint(&s[beg], (end - beg), length, cp)) {
-               out += cp;
-             }
-           });
-}
-
 }  // namespace utf16
 
 //-----------------------------------------------------------------------------