handle whatwg encoding standard overrides

compile_flags.txt

@@ -63,6 +63,7 @@
 -isystembazel-bin/src/rust/jsg/_virtual_includes/lib.rs@cxx
 -isystembazel-bin/src/rust/jsg/_virtual_includes/v8.rs@cxx
 -isystembazel-bin/src/rust/jsg-test/_virtual_includes/ffi-hdrs
+-isystembazel-bin/src/rust/encoding/_virtual_includes/lib.rs@cxx
 -isystembazel-bin/src/rust/jsg-test/_virtual_includes/lib.rs@cxx
 -isystembazel-bin/src/rust/gen-compile-cache/_virtual_includes/cxx-bridge
 -isystembazel-bin/src/rust/gen-compile-cache/_virtual_includes/gen-compile-cache@cxx

deps/rust/cargo.bzl

@@ -17,6 +17,7 @@ PACKAGES = WORKERD_CXX_PACKAGES | {
     "clang-ast": crate.spec(version = "0"),
     "clap": crate.spec(version = "4", default_features = False, features = ["derive", "std", "help"]),
     "codespan-reporting": crate.spec(version = "0"),
+    "encoding_rs": crate.spec(version = "0"),
     "flate2": crate.spec(version = "1"),
     "futures": crate.spec(version = "0"),
     "lol_html_c_api": crate.spec(git = "https://github.com/cloudflare/lol-html", tag = "v2.7.1"),

deps/rust/crates/BUILD.bazel

@@ -151,6 +151,18 @@ alias(
     tags = ["manual"],
 )
 
+alias(
+    name = "encoding_rs-0.8.35",
+    actual = "@crates_vendor__encoding_rs-0.8.35//:encoding_rs",
+    tags = ["manual"],
+)
+
+alias(
+    name = "encoding_rs",
+    actual = "@crates_vendor__encoding_rs-0.8.35//:encoding_rs",
+    tags = ["manual"],
+)
+
 alias(
     name = "flate2-1.1.9",
     actual = "@crates_vendor__flate2-1.1.9//:flate2",

deps/rust/crates/defs.bzl

@@ -305,6 +305,7 @@ _NORMAL_DEPENDENCIES = {
             "clang-ast": Label("@crates_vendor//:clang-ast-0.1.35"),
             "clap": Label("@crates_vendor//:clap-4.5.58"),
             "codespan-reporting": Label("@crates_vendor//:codespan-reporting-0.13.1"),
+            "encoding_rs": Label("@crates_vendor//:encoding_rs-0.8.35"),
             "flate2": Label("@crates_vendor//:flate2-1.1.9"),
             "foldhash": Label("@crates_vendor//:foldhash-0.2.0"),
             "futures": Label("@crates_vendor//:futures-0.3.31"),
@@ -2957,6 +2958,7 @@ def crate_repositories():
         struct(repo = "crates_vendor__clang-ast-0.1.35", is_dev_dep = False),
         struct(repo = "crates_vendor__clap-4.5.58", is_dev_dep = False),
         struct(repo = "crates_vendor__codespan-reporting-0.13.1", is_dev_dep = False),
+        struct(repo = "crates_vendor__encoding_rs-0.8.35", is_dev_dep = False),
         struct(repo = "crates_vendor__flate2-1.1.9", is_dev_dep = False),
         struct(repo = "crates_vendor__foldhash-0.2.0", is_dev_dep = False),
         struct(repo = "crates_vendor__futures-0.3.31", is_dev_dep = False),

src/rust/encoding/BUILD.bazel

@@ -0,0 +1,11 @@
+load("//:build/wd_rust_crate.bzl", "wd_rust_crate")
+
+wd_rust_crate(
+    name = "encoding",
+    cxx_bridge_src = "lib.rs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//src/rust/cxx-integration",
+        "@crates_vendor//:encoding_rs",
+    ],
+)

src/rust/encoding/lib.rs

@@ -0,0 +1,150 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+//! WHATWG Encoding Standard legacy decoders via `encoding_rs`.
+//!
+//! Exposes a streaming decoder to C++ via CXX bridge. All legacy encodings
+//! (CJK multi-byte, single-byte windows-1252, and x-user-defined) are handled
+//! by a single opaque `Decoder` type backed by `encoding_rs::Decoder`.
+
+#[cxx::bridge(namespace = "workerd::rust::encoding")]
+mod ffi {
+    /// Legacy encoding types supported by the Rust decoder.
+    /// Shared between C++ and Rust.
+    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+    #[repr(u16)]
+    enum Encoding {
+        Big5,
+        EucJp,
+        EucKr,
+        Gb18030,
+        Gbk,
+        Iso2022Jp,
+        ShiftJis,
+        Windows1252,
+        XUserDefined,
+    }
+
+    /// Result of a decode operation.
+    struct DecodeResult {
+        /// UTF-16 output.
+        output: Vec<u16>,
+        /// True if a fatal decoding error was encountered. Only meaningful
+        /// when the caller requested fatal mode — in replacement mode errors
+        /// are silently replaced with U+FFFD and this flag is not set.
+        had_error: bool,
+    }
+
+    extern "Rust" {
+        type Decoder;
+
+        /// Create a new streaming decoder for the given encoding.
+        // CXX bridge requires Box for opaque types.
+        #[expect(clippy::unnecessary_box_returns)]
+        fn new_decoder(encoding: Encoding) -> Box<Decoder>;
+
+        /// Decode a chunk of bytes. Set `flush` to true on the final chunk.
+        /// When `fatal` is true and an error is encountered, `had_error` is
+        /// set and the output may be incomplete.
+        fn decode(decoder: &mut Decoder, input: &[u8], flush: bool, fatal: bool) -> DecodeResult;
+
+        /// Reset the decoder to its initial state.
+        fn reset(decoder: &mut Decoder);
+    }
+}
+
+/// Opaque decoder state exposed to C++ via `Box<Decoder>`.
+pub struct Decoder {
+    encoding: &'static encoding_rs::Encoding,
+    inner: encoding_rs::Decoder,
+}
+
+/// Map a CXX-shared `Encoding` variant to the corresponding
+/// `encoding_rs` static.
+fn to_encoding(enc: ffi::Encoding) -> &'static encoding_rs::Encoding {
+    match enc {
+        ffi::Encoding::Big5 => encoding_rs::BIG5,
+        ffi::Encoding::EucJp => encoding_rs::EUC_JP,
+        ffi::Encoding::EucKr => encoding_rs::EUC_KR,
+        ffi::Encoding::Gb18030 => encoding_rs::GB18030,
+        ffi::Encoding::Gbk => encoding_rs::GBK,
+        ffi::Encoding::Iso2022Jp => encoding_rs::ISO_2022_JP,
+        ffi::Encoding::ShiftJis => encoding_rs::SHIFT_JIS,
+        ffi::Encoding::Windows1252 => encoding_rs::WINDOWS_1252,
+        ffi::Encoding::XUserDefined => encoding_rs::X_USER_DEFINED,
+        _ => unreachable!(),
+    }
+}
+
+pub fn new_decoder(encoding: ffi::Encoding) -> Box<Decoder> {
+    let encoding = to_encoding(encoding);
+    Box::new(Decoder {
+        inner: encoding.new_decoder_without_bom_handling(),
+        encoding,
+    })
+}
+
+pub fn decode(state: &mut Decoder, input: &[u8], flush: bool, fatal: bool) -> ffi::DecodeResult {
+    let max_len = state
+        .inner
+        .max_utf16_buffer_length(input.len())
+        .unwrap_or(input.len() + 4);
+    let mut output = vec![0u16; max_len];
+    let mut total_read = 0usize;
+    let mut total_written = 0usize;
+
+    if fatal {
+        loop {
+            let (result, read, written) = state.inner.decode_to_utf16_without_replacement(
+                &input[total_read..],
+                &mut output[total_written..],
+                flush,
+            );
+            total_read += read;
+            total_written += written;
+
+            match result {
+                encoding_rs::DecoderResult::InputEmpty => break,
+                encoding_rs::DecoderResult::OutputFull => {
+                    output.resize(output.len() * 2, 0);
+                }
+                encoding_rs::DecoderResult::Malformed(_, _) => {
+                    state.inner = state.encoding.new_decoder_without_bom_handling();
+                    output.truncate(total_written);
+                    return ffi::DecodeResult {
+                        output,
+                        had_error: true,
+                    };
+                }
+            }
+        }
+    } else {
+        loop {
+            let (result, read, written, _had_errors) = state.inner.decode_to_utf16(
+                &input[total_read..],
+                &mut output[total_written..],
+                flush,
+            );
+            total_read += read;
+            total_written += written;
+
+            match result {
+                encoding_rs::CoderResult::InputEmpty => break,
+                encoding_rs::CoderResult::OutputFull => {
+                    output.resize(output.len() * 2, 0);
+                }
+            }
+        }
+    }
+
+    output.truncate(total_written);
+    ffi::DecodeResult {
+        output,
+        had_error: false,
+    }
+}
+
+pub fn reset(state: &mut Decoder) {
+    state.inner = state.encoding.new_decoder_without_bom_handling();
+}

src/workerd/api/BUILD.bazel

@@ -430,19 +430,28 @@ wd_cc_library(
 
 wd_cc_library(
     name = "encoding",
-    srcs = ["encoding.c++"],
-    hdrs = ["encoding.h"],
+    srcs = [
+        "encoding.c++",
+        "encoding-legacy.c++",
+    ],
+    hdrs = [
+        "encoding.h",
+        "encoding-legacy.h",
+        "encoding-shared.h",
+    ],
     implementation_deps = [
         "//src/workerd/io:features",
         "//src/workerd/util:strings",
     ],
     visibility = ["//visibility:public"],
     deps = [
         ":util",
+        "//src/rust/encoding",
         "//src/workerd/io:compatibility-date_capnp",
         "//src/workerd/jsg",
         "@capnp-cpp//src/kj",
         "@simdutf",
+        "@workerd-cxx//kj-rs",
     ],
 )
 

src/workerd/api/encoding-legacy.c++

@@ -0,0 +1,75 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+#include "encoding-legacy.h"
+
+#include <kj-rs/convert.h>
+#include <rust/cxx.h>
+
+#include <kj/common.h>
+
+namespace workerd::api {
+
+namespace {
+
+// Map workerd::api::Encoding to the Rust-side RustEncoding enum.
+::workerd::rust::encoding::Encoding toRustEncoding(Encoding encoding) {
+  using RE = ::workerd::rust::encoding::Encoding;
+  switch (encoding) {
+    case Encoding::Big5:
+      return RE::Big5;
+    case Encoding::Euc_Jp:
+      return RE::EucJp;
+    case Encoding::Euc_Kr:
+      return RE::EucKr;
+    case Encoding::Gb18030:
+      return RE::Gb18030;
+    case Encoding::Gbk:
+      return RE::Gbk;
+    case Encoding::Iso2022_Jp:
+      return RE::Iso2022Jp;
+    case Encoding::Shift_Jis:
+      return RE::ShiftJis;
+    case Encoding::Windows_1252:
+      return RE::Windows1252;
+    case Encoding::X_User_Defined:
+      return RE::XUserDefined;
+    default:
+      KJ_UNREACHABLE;
+  }
+}
+
+}  // namespace
+
+LegacyDecoder::LegacyDecoder(Encoding encoding, DecoderFatal fatal)
+    : encoding(encoding),
+      fatal(fatal),
+      state(::workerd::rust::encoding::new_decoder(toRustEncoding(encoding))) {}
+
+void LegacyDecoder::reset() {
+  ::workerd::rust::encoding::reset(*state);
+}
+
+kj::Maybe<jsg::JsString> LegacyDecoder::decode(
+    jsg::Lock& js, kj::ArrayPtr<const kj::byte> buffer, bool flush) {
+  // Reset decoder state after flush, matching IcuDecoder's KJ_DEFER contract.
+  // This ensures decodePtr() (used by TextDecoderStream) resets correctly on flush.
+  KJ_DEFER({
+    if (flush) reset();
+  });
+
+  // kj_rs::RustMutable is used to avoid a copy of the underlying buffer.
+  auto result = ::workerd::rust::encoding::decode(
+      *state, buffer.as<kj_rs::RustMutable>(), flush, fatal.toBool());
+
+  if (fatal.toBool() && result.had_error) {
+    // Decoder state already reset by the Rust side on fatal error.
+    return kj::none;
+  }
+
+  auto output = kj::from<kj_rs::Rust>(result.output);
+  return js.str(output);
+}
+
+}  // namespace workerd::api

src/workerd/api/encoding-legacy.h

@@ -0,0 +1,51 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+// WHATWG-compliant legacy decoders (CJK multi-byte, windows-1252,
+// x-user-defined) implemented via the encoding_rs Rust crate through
+// a CXX bridge. A single LegacyDecoder class wraps an opaque Rust-side
+// decoder that handles all the encoding-specific state machines.
+
+#pragma once
+
+#include "encoding-shared.h"
+
+#include <workerd/rust/encoding/lib.rs.h>
+
+#include <rust/cxx.h>
+
+#include <kj/common.h>
+
+namespace workerd::api {
+
+// Unified legacy decoder using encoding_rs via Rust CXX bridge.
+// encoding_rs implements the full WHATWG decoder algorithms for all
+// legacy encodings, including streaming, error recovery, and ASCII
+// byte pushback.
+//
+// According to WHATWG spec, any encoding except UTF-8 and UTF-16 is considered legacy.
+class LegacyDecoder final: public Decoder {
+ public:
+  LegacyDecoder(Encoding encoding, DecoderFatal fatal);
+  ~LegacyDecoder() noexcept = default;
+  LegacyDecoder(LegacyDecoder&&) noexcept = default;
+  LegacyDecoder& operator=(LegacyDecoder&&) noexcept = default;
+  KJ_DISALLOW_COPY(LegacyDecoder);
+
+  Encoding getEncoding() override {
+    return encoding;
+  }
+
+  kj::Maybe<jsg::JsString> decode(
+      jsg::Lock& js, kj::ArrayPtr<const kj::byte> buffer, bool flush = false) override;
+
+  void reset() override;
+
+ private:
+  Encoding encoding;
+  DecoderFatal fatal;
+  ::rust::Box<::workerd::rust::encoding::Decoder> state;
+};
+
+}  // namespace workerd::api