From b77cb888b330b9b13287cf6193fe08a3d3e2341c Mon Sep 17 00:00:00 2001 From: Kumi Yasuda <119654600+ky438@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:59:03 +1000 Subject: [PATCH 1/3] Add zero-copy CBOR views --- include/jsoncons/source.hpp | 129 ++++ include/jsoncons_ext/cbor/cbor.hpp | 2 +- include/jsoncons_ext/cbor/cbor_parser.hpp | 153 ++++- include/jsoncons_ext/cbor/cbor_view.hpp | 732 ++++++++++++++++++++++ test/cbor/src/cbor_cursor_tests.cpp | 93 ++- 5 files changed, 1086 insertions(+), 23 deletions(-) create mode 100644 include/jsoncons_ext/cbor/cbor_view.hpp diff --git a/include/jsoncons/source.hpp b/include/jsoncons/source.hpp index a6ee9e7b5..2b44f3a3e 100644 --- a/include/jsoncons/source.hpp +++ b/include/jsoncons/source.hpp @@ -339,6 +339,28 @@ namespace jsoncons { return span(data, length); } + span read_span(std::size_t length) + { + if (length == 0) + { + return span(data_, std::size_t(0)); + } + if (length_ < length) + { + make_available(length); + } + if (length_ < length) + { + return span(); + } + + const value_type* data = data_; + data_ += length; + length_ -= length; + position_ += length; + return span(data, length); + } + std::size_t read(value_type* p, std::size_t length) { std::size_t len = 0; @@ -395,6 +417,63 @@ namespace jsoncons { } } private: + void reserve_buffer(std::size_t capacity) + { + if (capacity <= buffer_size_) + { + return; + } + + value_type* new_buffer = std::allocator_traits::allocate(alloc_, capacity); + if (length_ > 0) + { + std::memcpy(new_buffer, data_, sizeof(value_type)*length_); + } + if (buffer_) + { + std::allocator_traits::deallocate(alloc_, buffer_, buffer_size_); + } + buffer_ = new_buffer; + buffer_size_ = capacity; + data_ = buffer_; + } + + void make_available(std::size_t length) + { + if (length > buffer_size_) + { + reserve_buffer(length); + } + else if (length_ > 0 && data_ != buffer_) + { + std::memmove(buffer_, data_, sizeof(value_type)*length_); + data_ = buffer_; + } + else if (length_ == 0) + { + data_ = buffer_; + } + + while (length_ < length && !stream_ptr_->eof()) + { + JSONCONS_TRY + { + std::streamsize count = sbuf_->sgetn(reinterpret_cast(buffer_ + length_), buffer_size_ - length_); + std::size_t len = static_cast(count); + length_ += len; + if (len == 0 || length_ < length) + { + stream_ptr_->clear(stream_ptr_->rdstate() | std::ios::eofbit); + } + } + JSONCONS_CATCH(const std::exception&) + { + stream_ptr_->clear(stream_ptr_->rdstate() | std::ios::badbit | std::ios::eofbit); + return; + } + } + } + void fill_buffer() { if (stream_ptr_->eof()) @@ -503,6 +582,17 @@ namespace jsoncons { return span(data, length); } + span read_span(std::size_t length) + { + if (std::size_t(end_ - current_) < length) + { + return span(); + } + const value_type* data = current_; + current_ += length; + return span(data, length); + } + std::size_t read(value_type* p, std::size_t length) { std::size_t len; @@ -596,6 +686,20 @@ namespace jsoncons { return span(buffer_.data(), length); } + span read_span(std::size_t length) + { + if (length > buffer_.size()) + { + buffer_.resize(length); + } + std::size_t actual = read(buffer_.data(), length); + if (actual != length) + { + return span(); + } + return span(buffer_.data(), length); + } + template typename std::enable_if::value, std::size_t>::type read(value_type* data, std::size_t length) @@ -712,6 +816,17 @@ namespace jsoncons { return span(data, length); } + span read_span(std::size_t length) + { + if (std::size_t(end_ - current_) < length) + { + return span(); + } + const value_type* data = current_; + current_ += length; + return span(data, length); + } + std::size_t read(value_type* p, std::size_t length) { std::size_t len; @@ -803,6 +918,20 @@ namespace jsoncons { return span(buffer_.data(), length); } + span read_span(std::size_t length) + { + if (length > buffer_.size()) + { + buffer_.resize(length); + } + std::size_t actual = read(buffer_.data(), length); + if (actual != length) + { + return span(); + } + return span(buffer_.data(), length); + } + template typename std::enable_if::value, std::size_t>::type read(value_type* data, std::size_t length) diff --git a/include/jsoncons_ext/cbor/cbor.hpp b/include/jsoncons_ext/cbor/cbor.hpp index 107767fe7..7b8b6a27d 100644 --- a/include/jsoncons_ext/cbor/cbor.hpp +++ b/include/jsoncons_ext/cbor/cbor.hpp @@ -11,8 +11,8 @@ #include #include #include +#include #include #include #endif // JSONCONS_EXT_CBOR_CBOR_HPP - diff --git a/include/jsoncons_ext/cbor/cbor_parser.hpp b/include/jsoncons_ext/cbor/cbor_parser.hpp index ff7119f17..98e2cdf98 100644 --- a/include/jsoncons_ext/cbor/cbor_parser.hpp +++ b/include/jsoncons_ext/cbor/cbor_parser.hpp @@ -71,6 +71,7 @@ class basic_cbor_parser : public ser_context using tag_allocator_type = typename std::allocator_traits:: template rebind_alloc; using parse_state_allocator_type = typename std::allocator_traits:: template rebind_alloc; using string_type = std::basic_string; + using string_view_type = jsoncons::basic_string_view; using byte_string_type = std::vector; struct mapped_string @@ -86,6 +87,11 @@ class basic_cbor_parser : public ser_context { } + mapped_string(const string_view_type& sv, const allocator_type& alloc = allocator_type()) + : type(jsoncons::cbor::detail::cbor_major_type::text_string), str(sv.data(), sv.size(), alloc), bytes(alloc) + { + } + mapped_string(string_type&& str, const allocator_type& alloc = allocator_type()) : type(jsoncons::cbor::detail::cbor_major_type::text_string), str(std::move(str), alloc), bytes(alloc) { @@ -97,6 +103,12 @@ class basic_cbor_parser : public ser_context { } + mapped_string(const byte_string_view& bytes, + const allocator_type& alloc = allocator_type()) + : type(jsoncons::cbor::detail::cbor_major_type::byte_string), str(alloc), bytes(bytes.begin(),bytes.end(),alloc) + { + } + mapped_string(byte_string_type&& bytes, const allocator_type& alloc = allocator_type()) : type(jsoncons::cbor::detail::cbor_major_type::byte_string), str(alloc), bytes(std::move(bytes),alloc) @@ -171,6 +183,11 @@ class basic_cbor_parser : public ser_context c.push_back(b); } } + + byte_string_view view(std::error_code&) + { + return bytes; + } }; struct read_byte_string_from_source @@ -186,6 +203,11 @@ class basic_cbor_parser : public ser_context { source->read_byte_string(cont,ec); } + + byte_string_view view(std::error_code& ec) + { + return source->read_byte_string_view(ec); + } }; public: @@ -700,21 +722,19 @@ class basic_cbor_parser : public ser_context } case jsoncons::cbor::detail::cbor_major_type::text_string: { - text_buffer_.clear(); - - read_text_string(text_buffer_, ec); + auto sv = read_text_string_view(ec); if (JSONCONS_UNLIKELY(ec)) { return; } - auto result = unicode_traits::validate(text_buffer_.data(),text_buffer_.size()); + auto result = unicode_traits::validate(sv.data(),sv.size()); if (result.ec != unicode_traits::unicode_errc()) { ec = cbor_errc::invalid_utf8_text_string; more_ = false; return; } - handle_string(visitor, jsoncons::basic_string_view(text_buffer_.data(),text_buffer_.length()),ec); + handle_string(visitor, sv, ec); if (JSONCONS_UNLIKELY(ec)) { return; @@ -1093,6 +1113,52 @@ class basic_cbor_parser : public ser_context state_stack_.pop_back(); } + string_view_type read_text_string_view(std::error_code& ec) + { + auto c = source_.peek(); + if (JSONCONS_UNLIKELY(c.eof)) + { + ec = cbor_errc::unexpected_eof; + more_ = false; + return string_view_type(); + } + jsoncons::cbor::detail::cbor_major_type major_type = get_major_type(c.value); + JSONCONS_ASSERT(major_type == jsoncons::cbor::detail::cbor_major_type::text_string); + uint8_t info = get_additional_information_value(c.value); + + if (info == jsoncons::cbor::detail::additional_info::indefinite_length) + { + text_buffer_.clear(); + source_.ignore(1); + iterate_string_chunks(text_buffer_, major_type, ec); + if (JSONCONS_UNLIKELY(ec)) + { + return string_view_type(); + } + return string_view_type(text_buffer_.data(), text_buffer_.size()); + } + + std::size_t length = read_size(ec); + if (JSONCONS_UNLIKELY(ec)) + { + return string_view_type(); + } + auto data = source_.read_span(length); + if (data.size() != length) + { + ec = cbor_errc::unexpected_eof; + more_ = false; + return string_view_type(); + } + string_view_type sv(reinterpret_cast(data.data()), data.size()); + if (!stringref_map_stack_.empty() && + sv.length() >= jsoncons::cbor::detail::min_length_for_stringref(stringref_map_stack_.back().size())) + { + stringref_map_stack_.back().emplace_back(mapped_string(sv,alloc_)); + } + return sv; + } + void read_text_string(string_type& str, std::error_code& ec) { auto c = source_.peek(); @@ -1136,6 +1202,53 @@ class basic_cbor_parser : public ser_context } } + byte_string_view read_byte_string_view(std::error_code& ec) + { + auto c = source_.peek(); + if (JSONCONS_UNLIKELY(c.eof)) + { + ec = cbor_errc::unexpected_eof; + more_ = false; + return byte_string_view(); + } + jsoncons::cbor::detail::cbor_major_type major_type = get_major_type(c.value); + uint8_t info = get_additional_information_value(c.value); + + JSONCONS_ASSERT(major_type == jsoncons::cbor::detail::cbor_major_type::byte_string); + + if (info == jsoncons::cbor::detail::additional_info::indefinite_length) + { + bytes_buffer_.clear(); + source_.ignore(1); + iterate_string_chunks(bytes_buffer_, major_type, ec); + if (JSONCONS_UNLIKELY(ec)) + { + return byte_string_view(); + } + return byte_string_view(bytes_buffer_.data(), bytes_buffer_.size()); + } + + std::size_t length = read_size(ec); + if (JSONCONS_UNLIKELY(ec)) + { + return byte_string_view(); + } + auto data = source_.read_span(length); + if (data.size() != length) + { + ec = cbor_errc::unexpected_eof; + more_ = false; + return byte_string_view(); + } + byte_string_view bytes(data.data(), data.size()); + if (!stringref_map_stack_.empty() && + bytes.size() >= jsoncons::cbor::detail::min_length_for_stringref(stringref_map_stack_.back().size())) + { + stringref_map_stack_.back().emplace_back(mapped_string(bytes, alloc_)); + } + return bytes; + } + std::size_t read_size(std::error_code& ec) { uint64_t u = read_uint64(ec); @@ -1878,14 +1991,13 @@ class basic_cbor_parser : public ser_context { case 0x2: { - bytes_buffer_.clear(); - read(bytes_buffer_,ec); + auto bytes = read.view(ec); if (JSONCONS_UNLIKELY(ec)) { more_ = false; return; } - bigint n = bigint::from_bytes_be(1, bytes_buffer_.data(), bytes_buffer_.size()); + bigint n = bigint::from_bytes_be(1, bytes.data(), bytes.size()); text_buffer_.clear(); n.write_string(text_buffer_); visitor.string_value(text_buffer_, semantic_tag::bigint, *this, ec); @@ -1894,14 +2006,13 @@ class basic_cbor_parser : public ser_context } case 0x3: { - bytes_buffer_.clear(); - read(bytes_buffer_,ec); + auto bytes = read.view(ec); if (JSONCONS_UNLIKELY(ec)) { more_ = false; return; } - bigint n = bigint::from_bytes_be(1, bytes_buffer_.data(), bytes_buffer_.size()); + bigint n = bigint::from_bytes_be(1, bytes.data(), bytes.size()); n = -1 - n; text_buffer_.clear(); n.write_string(text_buffer_); @@ -1911,37 +2022,37 @@ class basic_cbor_parser : public ser_context } case 0x15: { - read(bytes_buffer_,ec); + auto bytes = read.view(ec); if (JSONCONS_UNLIKELY(ec)) { more_ = false; return; } - visitor.byte_string_value(bytes_buffer_, semantic_tag::base64url, *this, ec); + visitor.byte_string_value(bytes, semantic_tag::base64url, *this, ec); more_ = !cursor_mode_; break; } case 0x16: { - read(bytes_buffer_,ec); + auto bytes = read.view(ec); if (JSONCONS_UNLIKELY(ec)) { more_ = false; return; } - visitor.byte_string_value(bytes_buffer_, semantic_tag::base64, *this, ec); + visitor.byte_string_value(bytes, semantic_tag::base64, *this, ec); more_ = !cursor_mode_; break; } case 0x17: { - read(bytes_buffer_,ec); + auto bytes = read.view(ec); if (JSONCONS_UNLIKELY(ec)) { more_ = false; return; } - visitor.byte_string_value(bytes_buffer_, semantic_tag::base16, *this, ec); + visitor.byte_string_value(bytes, semantic_tag::base16, *this, ec); more_ = !cursor_mode_; break; } @@ -2457,13 +2568,13 @@ class basic_cbor_parser : public ser_context } default: { - read(bytes_buffer_,ec); + auto bytes = read.view(ec); if (JSONCONS_UNLIKELY(ec)) { more_ = false; return; } - visitor.byte_string_value(bytes_buffer_, raw_tag_, *this, ec); + visitor.byte_string_value(bytes, raw_tag_, *this, ec); more_ = !cursor_mode_; break; } @@ -2472,12 +2583,12 @@ class basic_cbor_parser : public ser_context } else { - read(bytes_buffer_,ec); + auto bytes = read.view(ec); if (JSONCONS_UNLIKELY(ec)) { return; } - visitor.byte_string_value(bytes_buffer_, semantic_tag::none, *this, ec); + visitor.byte_string_value(bytes, semantic_tag::none, *this, ec); more_ = !cursor_mode_; } } diff --git a/include/jsoncons_ext/cbor/cbor_view.hpp b/include/jsoncons_ext/cbor/cbor_view.hpp new file mode 100644 index 000000000..083eee7ac --- /dev/null +++ b/include/jsoncons_ext/cbor/cbor_view.hpp @@ -0,0 +1,732 @@ +// Copyright 2013-2026 Daniel Parker +// Distributed under the Boost license, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +// See https://github.com/danielaparker/jsoncons for latest version + +#ifndef JSONCONS_EXT_CBOR_CBOR_VIEW_HPP +#define JSONCONS_EXT_CBOR_CBOR_VIEW_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace jsoncons { +namespace cbor { +namespace view { + + enum class type_rank : uint8_t + { + null_value = 0, + false_value = 1, + true_value = 2, + undefined_value = 3, + simple_value = 4, + number_value = 5, + text_string = 6, + byte_string = 7, + array = 8, + map = 9, + tag = 10 + }; + + struct item_head + { + detail::cbor_major_type major_type{}; + uint8_t additional_info{0}; + uint64_t value{0}; + + bool indefinite() const noexcept + { + return additional_info == detail::additional_info::indefinite_length; + } + }; + + namespace detail_view { + + inline int sign(int value) noexcept + { + return (value > 0) - (value < 0); + } + + inline int compare_uint64(uint64_t a, uint64_t b) noexcept + { + return a < b ? -1 : (a > b ? 1 : 0); + } + + inline int compare_size(std::size_t a, std::size_t b) noexcept + { + return a < b ? -1 : (a > b ? 1 : 0); + } + + inline int compare_bytes(span a, span b) noexcept + { + const std::size_t n = (std::min)(a.size(), b.size()); + if (n != 0) + { + int r = std::memcmp(a.data(), b.data(), n); + if (r != 0) + { + return sign(r); + } + } + return compare_size(a.size(), b.size()); + } + + inline bool read_uint(const uint8_t*& p, const uint8_t* end, uint8_t info, uint64_t& value, std::error_code& ec) + { + if (info < 24) + { + value = info; + return true; + } + + std::size_t length = 0; + switch (info) + { + case 24: length = 1; break; + case 25: length = 2; break; + case 26: length = 4; break; + case 27: length = 8; break; + default: + ec = cbor_errc::unknown_type; + return false; + } + + if (static_cast(end - p) < length) + { + ec = cbor_errc::unexpected_eof; + return false; + } + + value = 0; + for (std::size_t i = 0; i < length; ++i) + { + value = (value << 8) | static_cast(p[i]); + } + p += length; + return true; + } + + inline bool read_head(const uint8_t*& p, const uint8_t* end, item_head& head, std::error_code& ec) + { + if (p >= end) + { + ec = cbor_errc::unexpected_eof; + return false; + } + + const uint8_t initial = *p++; + head.major_type = static_cast(initial >> 5); + head.additional_info = initial & 0x1f; + head.value = 0; + + if (head.additional_info == cbor::detail::additional_info::indefinite_length) + { + switch (head.major_type) + { + case cbor::detail::cbor_major_type::byte_string: + case cbor::detail::cbor_major_type::text_string: + case cbor::detail::cbor_major_type::array: + case cbor::detail::cbor_major_type::map: + break; + default: + ec = cbor_errc::unknown_type; + return false; + } + return true; + } + return read_uint(p, end, head.additional_info, head.value, ec); + } + + inline bool skip_item(const uint8_t*& p, const uint8_t* end, std::error_code& ec); + + inline bool skip_string_chunks(const uint8_t*& p, const uint8_t* end, + cbor::detail::cbor_major_type expected_major, std::error_code& ec) + { + for (;;) + { + if (p >= end) + { + ec = cbor_errc::unexpected_eof; + return false; + } + if (*p == 0xff) + { + ++p; + return true; + } + + item_head chunk; + if (!read_head(p, end, chunk, ec)) + { + return false; + } + if (chunk.major_type != expected_major || chunk.indefinite()) + { + ec = cbor_errc::illegal_chunked_string; + return false; + } + if (static_cast(end - p) < chunk.value) + { + ec = cbor_errc::unexpected_eof; + return false; + } + p += static_cast(chunk.value); + } + } + + inline bool skip_array_items(const uint8_t*& p, const uint8_t* end, const item_head& head, std::error_code& ec) + { + if (head.indefinite()) + { + for (;;) + { + if (p >= end) + { + ec = cbor_errc::unexpected_eof; + return false; + } + if (*p == 0xff) + { + ++p; + return true; + } + if (!skip_item(p, end, ec)) + { + return false; + } + } + } + + for (uint64_t i = 0; i < head.value; ++i) + { + if (!skip_item(p, end, ec)) + { + return false; + } + } + return true; + } + + inline bool skip_map_items(const uint8_t*& p, const uint8_t* end, const item_head& head, std::error_code& ec) + { + if (head.indefinite()) + { + for (;;) + { + if (p >= end) + { + ec = cbor_errc::unexpected_eof; + return false; + } + if (*p == 0xff) + { + ++p; + return true; + } + if (!skip_item(p, end, ec) || !skip_item(p, end, ec)) + { + return false; + } + } + } + + for (uint64_t i = 0; i < head.value; ++i) + { + if (!skip_item(p, end, ec) || !skip_item(p, end, ec)) + { + return false; + } + } + return true; + } + + inline bool skip_item(const uint8_t*& p, const uint8_t* end, std::error_code& ec) + { + item_head head; + if (!read_head(p, end, head, ec)) + { + return false; + } + + switch (head.major_type) + { + case cbor::detail::cbor_major_type::unsigned_integer: + case cbor::detail::cbor_major_type::negative_integer: + return true; + + case cbor::detail::cbor_major_type::simple: + if (head.indefinite()) + { + ec = cbor_errc::unknown_type; + return false; + } + return !head.indefinite(); + + case cbor::detail::cbor_major_type::byte_string: + case cbor::detail::cbor_major_type::text_string: + if (head.indefinite()) + { + return skip_string_chunks(p, end, head.major_type, ec); + } + if (static_cast(end - p) < head.value) + { + ec = cbor_errc::unexpected_eof; + return false; + } + p += static_cast(head.value); + return true; + + case cbor::detail::cbor_major_type::array: + return skip_array_items(p, end, head, ec); + + case cbor::detail::cbor_major_type::map: + return skip_map_items(p, end, head, ec); + + case cbor::detail::cbor_major_type::semantic_tag: + return skip_item(p, end, ec); + } + + ec = cbor_errc::unknown_type; + return false; + } + + inline span read_item_span(const uint8_t*& p, const uint8_t* end, std::error_code& ec) + { + const uint8_t* first = p; + if (!skip_item(p, end, ec)) + { + return span(); + } + return span(first, static_cast(p - first)); + } + + inline type_rank rank_for(const item_head& head) noexcept + { + switch (head.major_type) + { + case cbor::detail::cbor_major_type::unsigned_integer: + case cbor::detail::cbor_major_type::negative_integer: + return type_rank::number_value; + case cbor::detail::cbor_major_type::byte_string: + return type_rank::byte_string; + case cbor::detail::cbor_major_type::text_string: + return type_rank::text_string; + case cbor::detail::cbor_major_type::array: + return type_rank::array; + case cbor::detail::cbor_major_type::map: + return type_rank::map; + case cbor::detail::cbor_major_type::semantic_tag: + return type_rank::tag; + case cbor::detail::cbor_major_type::simple: + { + const uint64_t simple = head.additional_info == 24 ? head.value : head.additional_info; + switch (simple) + { + case 20: return type_rank::false_value; + case 21: return type_rank::true_value; + case 22: return type_rank::null_value; + case 23: return type_rank::undefined_value; + default: + return (head.additional_info == 25 || head.additional_info == 26 || head.additional_info == 27) + ? type_rank::number_value + : type_rank::simple_value; + } + } + } + return type_rank::simple_value; + } + + inline bool parse_number(span item, long double& value, std::error_code& ec) + { + const uint8_t* p = item.data(); + const uint8_t* end = p + item.size(); + item_head head; + if (!read_head(p, end, head, ec)) + { + return false; + } + + switch (head.major_type) + { + case cbor::detail::cbor_major_type::unsigned_integer: + value = static_cast(head.value); + return true; + + case cbor::detail::cbor_major_type::negative_integer: + value = -1.0L - static_cast(head.value); + return true; + + case cbor::detail::cbor_major_type::simple: + switch (head.additional_info) + { + case 25: + value = static_cast(binary::decode_half(static_cast(head.value))); + return true; + case 26: + { + uint32_t bits = static_cast(head.value); + float f; + std::memcpy(&f, &bits, sizeof(f)); + value = static_cast(f); + return true; + } + case 27: + { + uint64_t bits = head.value; + double d; + std::memcpy(&d, &bits, sizeof(d)); + value = static_cast(d); + return true; + } + default: + break; + } + break; + + default: + break; + } + + ec = cbor_errc::unknown_type; + return false; + } + + inline int compare_numbers(span a, span b, std::error_code& ec) + { + long double av = 0; + long double bv = 0; + if (!parse_number(a, av, ec) || !parse_number(b, bv, ec)) + { + return 0; + } + + const bool anan = std::isnan(av); + const bool bnan = std::isnan(bv); + if (anan || bnan) + { + return anan == bnan ? 0 : (anan ? -1 : 1); + } + if (av == 0.0L && bv == 0.0L) + { + return 0; + } + return av < bv ? -1 : (av > bv ? 1 : 0); + } + + inline bool definite_payload(span item, span& payload, std::error_code& ec) + { + const uint8_t* p = item.data(); + const uint8_t* end = p + item.size(); + item_head head; + if (!read_head(p, end, head, ec)) + { + return false; + } + if (head.indefinite()) + { + payload = item; + return false; + } + if (static_cast(end - p) < head.value) + { + ec = cbor_errc::unexpected_eof; + return false; + } + payload = span(p, static_cast(head.value)); + return true; + } + + inline int compare_items(span a, span b, std::error_code& ec); + + inline int compare_arrays(span a, span b, std::error_code& ec) + { + const uint8_t* ap = a.data(); + const uint8_t* bp = b.data(); + const uint8_t* aend = ap + a.size(); + const uint8_t* bend = bp + b.size(); + item_head ah; + item_head bh; + if (!read_head(ap, aend, ah, ec) || !read_head(bp, bend, bh, ec)) + { + return 0; + } + + uint64_t ai = 0; + uint64_t bi = 0; + for (;;) + { + if (ah.indefinite() && ap >= aend) + { + ec = cbor_errc::unexpected_eof; + return 0; + } + if (bh.indefinite() && bp >= bend) + { + ec = cbor_errc::unexpected_eof; + return 0; + } + const bool adone = ah.indefinite() ? (*ap == 0xff) : (ai == ah.value); + const bool bdone = bh.indefinite() ? (*bp == 0xff) : (bi == bh.value); + if (adone || bdone) + { + return adone == bdone ? 0 : (adone ? -1 : 1); + } + + auto av = read_item_span(ap, aend, ec); + if (ec) { return 0; } + auto bv = read_item_span(bp, bend, ec); + if (ec) { return 0; } + ++ai; + ++bi; + + int c = compare_items(av, bv, ec); + if (ec || c != 0) + { + return c; + } + } + } + + struct map_entry + { + span key; + span value; + }; + + inline bool read_map_entries(span item, std::vector& entries, std::error_code& ec) + { + const uint8_t* p = item.data(); + const uint8_t* end = p + item.size(); + item_head head; + if (!read_head(p, end, head, ec)) + { + return false; + } + if (head.major_type != cbor::detail::cbor_major_type::map) + { + ec = cbor_errc::unknown_type; + return false; + } + + if (!head.indefinite()) + { + if (static_cast(head.value) != head.value) + { + ec = cbor_errc::number_too_large; + return false; + } + entries.reserve(static_cast(head.value)); + for (uint64_t i = 0; i < head.value; ++i) + { + auto key = read_item_span(p, end, ec); + if (ec) { return false; } + auto value = read_item_span(p, end, ec); + if (ec) { return false; } + entries.push_back(map_entry{key, value}); + } + return true; + } + + for (;;) + { + if (p >= end) + { + ec = cbor_errc::unexpected_eof; + return false; + } + if (*p == 0xff) + { + ++p; + return true; + } + auto key = read_item_span(p, end, ec); + if (ec) { return false; } + auto value = read_item_span(p, end, ec); + if (ec) { return false; } + entries.push_back(map_entry{key, value}); + } + } + + inline int compare_maps(span a, span b, std::error_code& ec) + { + std::vector ae; + std::vector be; + if (!read_map_entries(a, ae, ec) || !read_map_entries(b, be, ec)) + { + return 0; + } + + auto less = [&ec](const map_entry& x, const map_entry& y) + { + int ck = compare_items(x.key, y.key, ec); + if (ec || ck != 0) + { + return ck < 0; + } + return compare_items(x.value, y.value, ec) < 0; + }; + std::sort(ae.begin(), ae.end(), less); + if (ec) { return 0; } + std::sort(be.begin(), be.end(), less); + if (ec) { return 0; } + + const std::size_t n = (std::min)(ae.size(), be.size()); + for (std::size_t i = 0; i < n; ++i) + { + int ck = compare_items(ae[i].key, be[i].key, ec); + if (ec || ck != 0) + { + return ck; + } + int cv = compare_items(ae[i].value, be[i].value, ec); + if (ec || cv != 0) + { + return cv; + } + } + return compare_size(ae.size(), be.size()); + } + + inline int compare_tags(span a, span b, std::error_code& ec) + { + const uint8_t* ap = a.data(); + const uint8_t* bp = b.data(); + const uint8_t* aend = ap + a.size(); + const uint8_t* bend = bp + b.size(); + item_head ah; + item_head bh; + if (!read_head(ap, aend, ah, ec) || !read_head(bp, bend, bh, ec)) + { + return 0; + } + int tag_compare = compare_uint64(ah.value, bh.value); + if (tag_compare != 0) + { + return tag_compare; + } + auto av = read_item_span(ap, aend, ec); + if (ec) { return 0; } + auto bv = read_item_span(bp, bend, ec); + if (ec) { return 0; } + return compare_items(av, bv, ec); + } + + inline int compare_items(span a, span b, std::error_code& ec) + { + if (a.size() == b.size() && compare_bytes(a, b) == 0) + { + return 0; + } + + const uint8_t* ap = a.data(); + const uint8_t* bp = b.data(); + item_head ah; + item_head bh; + if (!read_head(ap, ap + a.size(), ah, ec) || !read_head(bp, bp + b.size(), bh, ec)) + { + return 0; + } + + const type_rank ar = rank_for(ah); + const type_rank br = rank_for(bh); + if (ar != br) + { + return static_cast(ar) < static_cast(br) ? -1 : 1; + } + + switch (ar) + { + case type_rank::null_value: + case type_rank::false_value: + case type_rank::true_value: + case type_rank::undefined_value: + return 0; + + case type_rank::simple_value: + return compare_uint64(ah.additional_info == 24 ? ah.value : ah.additional_info, + bh.additional_info == 24 ? bh.value : bh.additional_info); + + case type_rank::number_value: + return compare_numbers(a, b, ec); + + case type_rank::text_string: + case type_rank::byte_string: + { + span av; + span bv; + const bool a_definite = definite_payload(a, av, ec); + if (ec) + { + return 0; + } + const bool b_definite = definite_payload(b, bv, ec); + if (ec) + { + return 0; + } + if (a_definite && b_definite) + { + return compare_bytes(av, bv); + } + return compare_bytes(a, b); + } + + case type_rank::array: + return compare_arrays(a, b, ec); + + case type_rank::map: + return compare_maps(a, b, ec); + + case type_rank::tag: + return compare_tags(a, b, ec); + } + return 0; + } + + } // namespace detail_view + + inline bool skip_item(const uint8_t*& p, const uint8_t* end, std::error_code& ec) + { + return detail_view::skip_item(p, end, ec); + } + + inline span item_span(span input, std::error_code& ec) + { + const uint8_t* p = input.data(); + return detail_view::read_item_span(p, p + input.size(), ec); + } + + inline int compare(span a, span b, std::error_code& ec) + { + return detail_view::compare_items(a, b, ec); + } + + inline int compare(span a, span b) + { + std::error_code ec; + int result = compare(a, b, ec); + if (JSONCONS_UNLIKELY(ec)) + { + JSONCONS_THROW(ser_error(ec)); + } + return result; + } + +} // namespace view +} // namespace cbor +} // namespace jsoncons + +#endif diff --git a/test/cbor/src/cbor_cursor_tests.cpp b/test/cbor/src/cbor_cursor_tests.cpp index 034c30b31..e24afa635 100644 --- a/test/cbor/src/cbor_cursor_tests.cpp +++ b/test/cbor/src/cbor_cursor_tests.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -19,6 +20,97 @@ using namespace jsoncons; +TEST_CASE("cbor cursor exposes definite strings as views") +{ + SECTION("text string from bytes source") + { + std::vector data = {'\x65','h','e','l','l','o'}; + cbor::cbor_bytes_cursor cursor(data); + + REQUIRE(staj_events::string_value == cursor.current().event_type()); + auto sv = cursor.current().get(); + CHECK(sv == jsoncons::string_view("hello", 5)); + CHECK(sv.data() == reinterpret_cast(data.data() + 1)); + } + + SECTION("byte string from bytes source") + { + std::vector data = {'\x43',0x01,0x02,0x03}; + cbor::cbor_bytes_cursor cursor(data); + + REQUIRE(staj_events::byte_string_value == cursor.current().event_type()); + auto bytes = cursor.current().get(); + CHECK(bytes.size() == 3); + CHECK(bytes.data() == data.data() + 1); + CHECK(bytes[0] == 0x01); + CHECK(bytes[2] == 0x03); + } +} + +TEST_CASE("cbor stream source spans straddled strings") +{ + std::string data; + data.push_back('\x65'); + data.append("hello"); + std::istringstream is(data); + jsoncons::binary_stream_source source(is, 2); + std::error_code ec; + cbor::cbor_stream_cursor cursor(std::move(source), ec); + + REQUIRE_FALSE(ec); + REQUIRE(staj_events::string_value == cursor.current().event_type()); + CHECK(cursor.current().get() == "hello"); +} + +TEST_CASE("cbor raw view utilities") +{ + SECTION("item span reads one item") + { + std::vector data = {0x01,0x02}; + std::error_code ec; + auto item = cbor::view::item_span(jsoncons::span(data), ec); + REQUIRE_FALSE(ec); + REQUIRE(item.size() == 1); + CHECK(item[0] == 0x01); + } + + SECTION("numbers share one collation rank") + { + std::vector one = {0x01}; + std::vector one_double = {0xfb,0x3f,0xf0,0x00,0x00,0x00,0x00,0x00,0x00}; + std::error_code ec; + CHECK(cbor::view::compare(jsoncons::span(one), jsoncons::span(one_double), ec) == 0); + CHECK_FALSE(ec); + } + + SECTION("arrays compare lexicographically") + { + std::vector a = {0x81,0x01}; + std::vector b = {0x82,0x01,0x00}; + std::error_code ec; + CHECK(cbor::view::compare(jsoncons::span(a), jsoncons::span(b), ec) < 0); + CHECK_FALSE(ec); + } + + SECTION("maps are key-order independent") + { + std::vector m1 = {0xa2,0x01,0x61,0x61,0x61,0x6b,0x02}; + std::vector m2 = {0xa2,0x61,0x6b,0x02,0x01,0x61,0x61}; + std::error_code ec; + CHECK(cbor::view::compare(jsoncons::span(m1), jsoncons::span(m2), ec) == 0); + CHECK_FALSE(ec); + } + + SECTION("invalid indefinite integer is rejected") + { + std::vector data = {0x1f}; + std::error_code ec; + auto item = cbor::view::item_span(jsoncons::span(data), ec); + CHECK(item.empty()); + CHECK(ec == cbor::cbor_errc::unknown_type); + } +} + TEST_CASE("cbor_cursor reputon test") { ojson j = ojson::parse(R"( @@ -572,4 +664,3 @@ TEMPLATE_TEST_CASE("cbor_event_reader reset test", "", CHECK(reader.done()); } } - From 58575c9745096438c41e1b036481a368581a60fa Mon Sep 17 00:00:00 2001 From: Kumi Yasuda <119654600+ky438@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:26:26 +1000 Subject: [PATCH 2/3] Improve CBOR view collation and bulk writes --- include/jsoncons/sink.hpp | 43 +++++ include/jsoncons_ext/cbor/cbor_encoder.hpp | 10 +- include/jsoncons_ext/cbor/cbor_view.hpp | 209 ++++++++++++++++----- test/cbor/src/cbor_cursor_tests.cpp | 31 +++ 4 files changed, 239 insertions(+), 54 deletions(-) diff --git a/include/jsoncons/sink.hpp b/include/jsoncons/sink.hpp index fff11c61d..59636cf6f 100644 --- a/include/jsoncons/sink.hpp +++ b/include/jsoncons/sink.hpp @@ -325,10 +325,53 @@ namespace jsoncons { { } + void append(const uint8_t* s, std::size_t length) + { + append_impl(*buf_ptr, s, length, 0); + } + void push_back(uint8_t ch) { buf_ptr->push_back(static_cast(ch)); } + + private: + template + static auto append_impl(C& c, const uint8_t* s, std::size_t length, int) + -> decltype(c.append(s, length), void()) + { + c.append(s, length); + } + + template + static auto append_impl(C& c, const uint8_t* s, std::size_t length, long) + -> decltype(c.insert(c.end(), s, s + length), void()) + { + reserve(c, length, 0); + c.insert(c.end(), s, s + length); + } + + template + static void append_impl(C& c, const uint8_t* s, std::size_t length, ...) + { + reserve(c, length, 0); + for (std::size_t i = 0; i < length; ++i) + { + c.push_back(static_cast(s[i])); + } + } + + template + static auto reserve(C& c, std::size_t length, int) + -> decltype(c.reserve(c.size() + length), void()) + { + c.reserve(c.size() + length); + } + + template + static void reserve(C&, std::size_t, ...) + { + } }; } // namespace jsoncons diff --git a/include/jsoncons_ext/cbor/cbor_encoder.hpp b/include/jsoncons_ext/cbor/cbor_encoder.hpp index 043b45ca5..984407e3f 100644 --- a/include/jsoncons_ext/cbor/cbor_encoder.hpp +++ b/include/jsoncons_ext/cbor/cbor_encoder.hpp @@ -524,10 +524,7 @@ class basic_cbor_encoder final : public basic_json_visitor std::back_inserter(sink_)); } - for (auto c : sv) - { - sink_.push_back(c); - } + sink_.append(reinterpret_cast(sv.data()), sv.size()); } void write_bignum(bigint& n) @@ -1104,10 +1101,7 @@ class basic_cbor_encoder final : public basic_json_visitor std::back_inserter(sink_)); } - for (auto c : b) - { - sink_.push_back(c); - } + sink_.append(b.data(), b.size()); } JSONCONS_VISITOR_RETURN_TYPE visit_double(double val, diff --git a/include/jsoncons_ext/cbor/cbor_view.hpp b/include/jsoncons_ext/cbor/cbor_view.hpp index 083eee7ac..5bbcc9b71 100644 --- a/include/jsoncons_ext/cbor/cbor_view.hpp +++ b/include/jsoncons_ext/cbor/cbor_view.hpp @@ -53,6 +53,12 @@ namespace view { } }; + struct map_entry_view + { + span key; + span value; + }; + namespace detail_view { inline int sign(int value) noexcept @@ -426,27 +432,156 @@ namespace view { return av < bv ? -1 : (av > bv ? 1 : 0); } - inline bool definite_payload(span item, span& payload, std::error_code& ec) + class string_chunk_reader { - const uint8_t* p = item.data(); - const uint8_t* end = p + item.size(); - item_head head; - if (!read_head(p, end, head, ec)) - { - return false; + cbor::detail::cbor_major_type major_type_{}; + const uint8_t* p_{nullptr}; + const uint8_t* end_{nullptr}; + span payload_; + bool indefinite_{false}; + bool done_{false}; + + public: + string_chunk_reader(span item, std::error_code& ec) + { + p_ = item.data(); + end_ = p_ + item.size(); + item_head head; + if (!read_head(p_, end_, head, ec)) + { + done_ = true; + return; + } + major_type_ = head.major_type; + if (major_type_ != cbor::detail::cbor_major_type::byte_string && + major_type_ != cbor::detail::cbor_major_type::text_string) + { + ec = cbor_errc::unknown_type; + done_ = true; + return; + } + indefinite_ = head.indefinite(); + if (!indefinite_) + { + if (static_cast(end_ - p_) < head.value) + { + ec = cbor_errc::unexpected_eof; + done_ = true; + return; + } + payload_ = span(p_, static_cast(head.value)); + p_ += static_cast(head.value); + } } - if (head.indefinite()) + + bool next(span& chunk, std::error_code& ec) { - payload = item; - return false; + chunk = span(); + if (done_) + { + return false; + } + if (!indefinite_) + { + done_ = true; + chunk = payload_; + return true; + } + + for (;;) + { + if (p_ >= end_) + { + ec = cbor_errc::unexpected_eof; + done_ = true; + return false; + } + if (*p_ == 0xff) + { + ++p_; + done_ = true; + return false; + } + + item_head head; + if (!read_head(p_, end_, head, ec)) + { + done_ = true; + return false; + } + if (head.major_type != major_type_ || head.indefinite()) + { + ec = cbor_errc::illegal_chunked_string; + done_ = true; + return false; + } + if (static_cast(end_ - p_) < head.value) + { + ec = cbor_errc::unexpected_eof; + done_ = true; + return false; + } + chunk = span(p_, static_cast(head.value)); + p_ += static_cast(head.value); + return true; + } } - if (static_cast(end - p) < head.value) + }; + + inline int compare_string_items(span a, span b, std::error_code& ec) + { + string_chunk_reader ar(a, ec); + if (ec) { return 0; } + string_chunk_reader br(b, ec); + if (ec) { return 0; } + + span ac; + span bc; + std::size_t ai = 0; + std::size_t bi = 0; + bool ahas = false; + bool bhas = false; + + for (;;) { - ec = cbor_errc::unexpected_eof; - return false; + while (!ahas || ai == ac.size()) + { + ahas = ar.next(ac, ec); + if (ec) { return 0; } + ai = 0; + if (!ahas || ac.size() != 0) + { + break; + } + } + while (!bhas || bi == bc.size()) + { + bhas = br.next(bc, ec); + if (ec) { return 0; } + bi = 0; + if (!bhas || bc.size() != 0) + { + break; + } + } + + if (!ahas || !bhas) + { + return ahas == bhas ? 0 : (ahas ? 1 : -1); + } + + const std::size_t n = (std::min)(ac.size() - ai, bc.size() - bi); + if (n != 0) + { + int r = std::memcmp(ac.data() + ai, bc.data() + bi, n); + if (r != 0) + { + return sign(r); + } + ai += n; + bi += n; + } } - payload = span(p, static_cast(head.value)); - return true; } inline int compare_items(span a, span b, std::error_code& ec); @@ -500,13 +635,7 @@ namespace view { } } - struct map_entry - { - span key; - span value; - }; - - inline bool read_map_entries(span item, std::vector& entries, std::error_code& ec) + inline bool read_map_entries(span item, std::vector& entries, std::error_code& ec) { const uint8_t* p = item.data(); const uint8_t* end = p + item.size(); @@ -535,7 +664,7 @@ namespace view { if (ec) { return false; } auto value = read_item_span(p, end, ec); if (ec) { return false; } - entries.push_back(map_entry{key, value}); + entries.push_back(map_entry_view{key, value}); } return true; } @@ -556,20 +685,20 @@ namespace view { if (ec) { return false; } auto value = read_item_span(p, end, ec); if (ec) { return false; } - entries.push_back(map_entry{key, value}); + entries.push_back(map_entry_view{key, value}); } } inline int compare_maps(span a, span b, std::error_code& ec) { - std::vector ae; - std::vector be; + std::vector ae; + std::vector be; if (!read_map_entries(a, ae, ec) || !read_map_entries(b, be, ec)) { return 0; } - auto less = [&ec](const map_entry& x, const map_entry& y) + auto less = [&ec](const map_entry_view& x, const map_entry_view& y) { int ck = compare_items(x.key, y.key, ec); if (ec || ck != 0) @@ -664,25 +793,7 @@ namespace view { case type_rank::text_string: case type_rank::byte_string: - { - span av; - span bv; - const bool a_definite = definite_payload(a, av, ec); - if (ec) - { - return 0; - } - const bool b_definite = definite_payload(b, bv, ec); - if (ec) - { - return 0; - } - if (a_definite && b_definite) - { - return compare_bytes(av, bv); - } - return compare_bytes(a, b); - } + return compare_string_items(a, b, ec); case type_rank::array: return compare_arrays(a, b, ec); @@ -709,6 +820,12 @@ namespace view { return detail_view::read_item_span(p, p + input.size(), ec); } + inline bool map_entries(span input, std::vector& entries, std::error_code& ec) + { + entries.clear(); + return detail_view::read_map_entries(input, entries, ec); + } + inline int compare(span a, span b, std::error_code& ec) { return detail_view::compare_items(a, b, ec); diff --git a/test/cbor/src/cbor_cursor_tests.cpp b/test/cbor/src/cbor_cursor_tests.cpp index e24afa635..be1e57310 100644 --- a/test/cbor/src/cbor_cursor_tests.cpp +++ b/test/cbor/src/cbor_cursor_tests.cpp @@ -101,6 +101,37 @@ TEST_CASE("cbor raw view utilities") CHECK_FALSE(ec); } + SECTION("map entries expose key and value spans") + { + std::vector data = {0xa2,0x01,0x61,0x61,0x61,0x6b,0x02}; + std::vector entries; + std::error_code ec; + REQUIRE(cbor::view::map_entries(jsoncons::span(data), entries, ec)); + REQUIRE_FALSE(ec); + REQUIRE(entries.size() == 2); + CHECK(entries[0].key.data() == data.data() + 1); + CHECK(entries[0].key.size() == 1); + CHECK(entries[0].value.data() == data.data() + 2); + CHECK(entries[0].value.size() == 2); + CHECK(entries[1].key.data() == data.data() + 4); + CHECK(entries[1].key.size() == 2); + CHECK(entries[1].value.data() == data.data() + 6); + CHECK(entries[1].value.size() == 1); + } + + SECTION("indefinite strings compare semantically") + { + std::vector definite_text = {0x65,'h','e','l','l','o'}; + std::vector chunked_text = {0x7f,0x61,'h',0x64,'e','l','l','o',0xff}; + std::vector definite_bytes = {0x42,0x01,0x02}; + std::vector chunked_bytes = {0x5f,0x41,0x01,0x41,0x02,0xff}; + std::error_code ec; + CHECK(cbor::view::compare(jsoncons::span(definite_text), jsoncons::span(chunked_text), ec) == 0); + REQUIRE_FALSE(ec); + CHECK(cbor::view::compare(jsoncons::span(definite_bytes), jsoncons::span(chunked_bytes), ec) == 0); + REQUIRE_FALSE(ec); + } + SECTION("invalid indefinite integer is rejected") { std::vector data = {0x1f}; From 3e0b84f99c7640eb06c91799829f9c4feb2494b4 Mon Sep 17 00:00:00 2001 From: Kumi Yasuda <119654600+ky438@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:46:38 +1000 Subject: [PATCH 3/3] Bound stream CBOR spans and simplify length writes --- include/jsoncons/sink.hpp | 1 - include/jsoncons/source.hpp | 4 ++ include/jsoncons_ext/cbor/cbor_encoder.hpp | 55 +++++----------------- include/jsoncons_ext/cbor/cbor_parser.hpp | 36 +++++++++++--- test/cbor/src/cbor_cursor_tests.cpp | 10 ++++ 5 files changed, 56 insertions(+), 50 deletions(-) diff --git a/include/jsoncons/sink.hpp b/include/jsoncons/sink.hpp index 59636cf6f..0461823bd 100644 --- a/include/jsoncons/sink.hpp +++ b/include/jsoncons/sink.hpp @@ -347,7 +347,6 @@ namespace jsoncons { static auto append_impl(C& c, const uint8_t* s, std::size_t length, long) -> decltype(c.insert(c.end(), s, s + length), void()) { - reserve(c, length, 0); c.insert(c.end(), s, s + length); } diff --git a/include/jsoncons/source.hpp b/include/jsoncons/source.hpp index 2b44f3a3e..cd8e7b42a 100644 --- a/include/jsoncons/source.hpp +++ b/include/jsoncons/source.hpp @@ -442,6 +442,10 @@ namespace jsoncons { { if (length > buffer_size_) { + if (length > default_max_buffer_size) + { + return; + } reserve_buffer(length); } else if (length_ > 0 && data_ != buffer_) diff --git a/include/jsoncons_ext/cbor/cbor_encoder.hpp b/include/jsoncons_ext/cbor/cbor_encoder.hpp index 984407e3f..7ab7a6126 100644 --- a/include/jsoncons_ext/cbor/cbor_encoder.hpp +++ b/include/jsoncons_ext/cbor/cbor_encoder.hpp @@ -485,44 +485,46 @@ class basic_cbor_encoder final : public basic_json_visitor } } - void write_utf8_string(const string_view& sv) + void write_type_and_length(uint8_t major_type, std::size_t length) { - const size_t length = sv.size(); - if (length <= 0x17) { - // fixstr stores a byte array whose length is upto 31 bytes - binary::native_to_big(static_cast(0x60 + length), + binary::native_to_big(static_cast(major_type + length), std::back_inserter(sink_)); } else if (length <= 0xff) { - binary::native_to_big(static_cast(0x78), + binary::native_to_big(static_cast(major_type + 0x18), std::back_inserter(sink_)); binary::native_to_big(static_cast(length), std::back_inserter(sink_)); } else if (length <= 0xffff) { - binary::native_to_big(static_cast(0x79), + binary::native_to_big(static_cast(major_type + 0x19), std::back_inserter(sink_)); binary::native_to_big(static_cast(length), std::back_inserter(sink_)); } else if (length <= 0xffffffff) { - binary::native_to_big(static_cast(0x7a), + binary::native_to_big(static_cast(major_type + 0x1a), std::back_inserter(sink_)); binary::native_to_big(static_cast(length), std::back_inserter(sink_)); } else if (uint64_t(length) <= (std::numeric_limits::max)()) { - binary::native_to_big(static_cast(0x7b), + binary::native_to_big(static_cast(major_type + 0x1b), std::back_inserter(sink_)); binary::native_to_big(static_cast(length), std::back_inserter(sink_)); } + } + + void write_utf8_string(const string_view& sv) + { + write_type_and_length(0x60, sv.size()); sink_.append(reinterpret_cast(sv.data()), sv.size()); } @@ -1066,40 +1068,7 @@ class basic_cbor_encoder final : public basic_json_visitor void write_byte_string(const byte_string_view& b) { - if (b.size() <= 0x17) - { - // fixstr stores a byte array whose length is upto 31 bytes - binary::native_to_big(static_cast(0x40 + b.size()), - std::back_inserter(sink_)); - } - else if (b.size() <= 0xff) - { - binary::native_to_big(static_cast(0x58), - std::back_inserter(sink_)); - binary::native_to_big(static_cast(b.size()), - std::back_inserter(sink_)); - } - else if (b.size() <= 0xffff) - { - binary::native_to_big(static_cast(0x59), - std::back_inserter(sink_)); - binary::native_to_big(static_cast(b.size()), - std::back_inserter(sink_)); - } - else if (b.size() <= 0xffffffff) - { - binary::native_to_big(static_cast(0x5a), - std::back_inserter(sink_)); - binary::native_to_big(static_cast(b.size()), - std::back_inserter(sink_)); - } - else // if (b.size() <= 0xffffffffffffffff) - { - binary::native_to_big(static_cast(0x5b), - std::back_inserter(sink_)); - binary::native_to_big(static_cast(b.size()), - std::back_inserter(sink_)); - } + write_type_and_length(0x40, b.size()); sink_.append(b.data(), b.size()); } diff --git a/include/jsoncons_ext/cbor/cbor_parser.hpp b/include/jsoncons_ext/cbor/cbor_parser.hpp index 98e2cdf98..d72bbbaf5 100644 --- a/include/jsoncons_ext/cbor/cbor_parser.hpp +++ b/include/jsoncons_ext/cbor/cbor_parser.hpp @@ -1144,17 +1144,29 @@ class basic_cbor_parser : public ser_context return string_view_type(); } auto data = source_.read_span(length); - if (data.size() != length) + if (data.size() == length) + { + string_view_type sv(reinterpret_cast(data.data()), data.size()); + if (!stringref_map_stack_.empty() && + sv.length() >= jsoncons::cbor::detail::min_length_for_stringref(stringref_map_stack_.back().size())) + { + stringref_map_stack_.back().emplace_back(mapped_string(sv,alloc_)); + } + return sv; + } + + text_buffer_.clear(); + if (source_reader::read(source_, text_buffer_, length) != length) { ec = cbor_errc::unexpected_eof; more_ = false; return string_view_type(); } - string_view_type sv(reinterpret_cast(data.data()), data.size()); + string_view_type sv(text_buffer_.data(), text_buffer_.size()); if (!stringref_map_stack_.empty() && sv.length() >= jsoncons::cbor::detail::min_length_for_stringref(stringref_map_stack_.back().size())) { - stringref_map_stack_.back().emplace_back(mapped_string(sv,alloc_)); + stringref_map_stack_.back().emplace_back(mapped_string(text_buffer_,alloc_)); } return sv; } @@ -1234,17 +1246,29 @@ class basic_cbor_parser : public ser_context return byte_string_view(); } auto data = source_.read_span(length); - if (data.size() != length) + if (data.size() == length) + { + byte_string_view bytes(data.data(), data.size()); + if (!stringref_map_stack_.empty() && + bytes.size() >= jsoncons::cbor::detail::min_length_for_stringref(stringref_map_stack_.back().size())) + { + stringref_map_stack_.back().emplace_back(mapped_string(bytes, alloc_)); + } + return bytes; + } + + bytes_buffer_.clear(); + if (source_reader::read(source_, bytes_buffer_, length) != length) { ec = cbor_errc::unexpected_eof; more_ = false; return byte_string_view(); } - byte_string_view bytes(data.data(), data.size()); + byte_string_view bytes(bytes_buffer_.data(), bytes_buffer_.size()); if (!stringref_map_stack_.empty() && bytes.size() >= jsoncons::cbor::detail::min_length_for_stringref(stringref_map_stack_.back().size())) { - stringref_map_stack_.back().emplace_back(mapped_string(bytes, alloc_)); + stringref_map_stack_.back().emplace_back(mapped_string(bytes_buffer_, alloc_)); } return bytes; } diff --git a/test/cbor/src/cbor_cursor_tests.cpp b/test/cbor/src/cbor_cursor_tests.cpp index be1e57310..06814b0a7 100644 --- a/test/cbor/src/cbor_cursor_tests.cpp +++ b/test/cbor/src/cbor_cursor_tests.cpp @@ -62,6 +62,16 @@ TEST_CASE("cbor stream source spans straddled strings") CHECK(cursor.current().get() == "hello"); } +TEST_CASE("cbor stream rejects truncated huge byte string length") +{ + std::string data; + data.push_back('\x5b'); + data.push_back('\xf6'); + std::istringstream is(data); + + CHECK_THROWS_AS(cbor::decode_cbor(is), ser_error); +} + TEST_CASE("cbor raw view utilities") { SECTION("item span reads one item")