From 76a3cf8178d5a5dfb51bea8ad2fcb30111d67872 Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Thu, 26 Feb 2026 11:56:13 +0100 Subject: [PATCH 1/5] clang: remove clang18 fallback code This code did not properly reset errno anyway which is why it broke some unittests. Since clang<20 is not supported anymore anyway, I choose to remove it instead Signed-off-by: Alexander Krimm --- src/core/include/opencmw.hpp | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/src/core/include/opencmw.hpp b/src/core/include/opencmw.hpp index 50b2ac93..4c2d0a9a 100644 --- a/src/core/include/opencmw.hpp +++ b/src/core/include/opencmw.hpp @@ -779,47 +779,11 @@ inline constexpr void diffView(std::ostream &os, const T &lhs, const T &rhs) { template static std::errc parseNumber(std::string_view str, T &number) { - // wrapper for missing std::from_chars() support for floating point types in clang < 20 - // Note that this implementation is intolerant to trailing non-number junk in the given string. -#ifdef __clang__ - if constexpr (std::is_same_v) { - char *endPtr = nullptr; - float value = std::strtof(str.data(), &endPtr); - if (endPtr == str.data() + str.size() && errno == 0) { - number = value; - return std::errc{}; - } - return std::errc::invalid_argument; - } else if constexpr (std::is_same_v) { - char *endPtr = nullptr; - double value = std::strtod(str.data(), &endPtr); - if (endPtr == str.data() + str.size() && errno == 0) { - number = value; - return std::errc{}; - } - return std::errc::invalid_argument; - } else if constexpr (std::is_same_v) { - char *endPtr = nullptr; - long double value = std::strtold(str.data(), &endPtr); - if (endPtr == str.data() + str.size() && errno == 0) { - number = value; - return std::errc{}; - } - return std::errc::invalid_argument; - } else { - const auto rc = std::from_chars(str.data(), str.data() + str.size(), number); - if (rc.ptr == str.data() + str.size() && rc.ec == std::errc{}) { - return std::errc{}; - } - return rc.ec == std::errc{} ? std::errc::invalid_argument : rc.ec; - } -#else const auto rc = std::from_chars(str.data(), str.data() + str.size(), number); if (rc.ptr == str.data() + str.size() && rc.ec == std::errc{}) { return std::errc{}; } return rc.ec == std::errc{} ? std::errc::invalid_argument : rc.ec; -#endif } } // namespace opencmw From c65590fd87e95c3991fc68d3722e4d02b3e9cebe Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Mon, 16 Feb 2026 15:07:47 +0100 Subject: [PATCH 2/5] SpinLock: fix missing header Signed-off-by: Alexander Krimm --- src/core/include/SpinWait.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/core/include/SpinWait.hpp b/src/core/include/SpinWait.hpp index df0bd136..1ab2aa66 100644 --- a/src/core/include/SpinWait.hpp +++ b/src/core/include/SpinWait.hpp @@ -1,6 +1,7 @@ #ifndef SPIN_WAIT_HPP #define SPIN_WAIT_HPP +#include #include #include #include @@ -67,12 +68,12 @@ class SpinWait { void reset() noexcept { _count = 0; } template - requires std::is_nothrow_invocable_r_v + requires std::is_nothrow_invocable_r_v bool spinUntil(const T &condition) const { return spinUntil(condition, -1); } template - requires std::is_nothrow_invocable_r_v + requires std::is_nothrow_invocable_r_v bool spinUntil(const T &condition, std::int64_t millisecondsTimeout) const { if (millisecondsTimeout < -1) { @@ -111,8 +112,8 @@ class AtomicMutex { SPIN_WAIT _spin_wait; public: - AtomicMutex() = default; - AtomicMutex(const AtomicMutex &) = delete; + AtomicMutex() = default; + AtomicMutex(const AtomicMutex &) = delete; AtomicMutex &operator=(const AtomicMutex &) = delete; // From 9f058e169aa9d3a1e2d6c3882e8fed1a5343784a Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Mon, 1 Jul 2024 17:40:35 +0200 Subject: [PATCH 3/5] CMWLightSerialiser: add features - Enable support for std::map and nesting Support for nested Datastructures was not implemented correctly before because only serialise + deserialise was checked but not the actual serialised output. - fix handling of empty nested objects/maps - add std::variant serialiser Serialisation only, as deserialisation is not possible due to the IoSerialiser expecting to be able to deduce the type of the field at compile time via `IoSerialiser::getTypeId()`. - Allows fieldnames with starting with a numeric value by prefixing them with ' x_'. - IoSerialiserCmwLight: fix array (de)serialisation Fixes two compatibility issues in the cmw light serialisation format. - The branches for skipping bool arrays that don't exist in the target object were missing => added additional branches. - CMW always serialises n_dims, [nx, ...], n_elem, [x1, ...] even for 1D arrays. Before only the number of elements was (de)serialised leading to incompatible serialised data and crashes on deserialisation. Also adds a testcase which exercises these field types and apply fixes. Signed-off-by: Alexander Krimm --- src/serialiser/include/IoSerialiser.hpp | 20 +- .../include/IoSerialiserCmwLight.hpp | 188 ++++++-- .../test/IoSerialiserCmwLight_tests.cpp | 445 +++++++++++++++++- 3 files changed, 591 insertions(+), 62 deletions(-) diff --git a/src/serialiser/include/IoSerialiser.hpp b/src/serialiser/include/IoSerialiser.hpp index 91f0e64a..570460f6 100644 --- a/src/serialiser/include/IoSerialiser.hpp +++ b/src/serialiser/include/IoSerialiser.hpp @@ -147,10 +147,18 @@ struct IoSerialiser { } }; +template string> +constexpr const char *sanitizeFieldName() { + if constexpr (N > 2 && string.data[0] == 'x' && string.data[1] == '_') { + return string.c_str() + 2; + } + return string.c_str(); +} + template OPENCMW_FORCEINLINE int32_t findMemberIndex(const std::string_view &fieldName) noexcept { static constexpr ConstExprMap().members.size> m{ refl::util::map_to_array>(refl::reflect().members, [](auto field, auto index) { - return std::pair(field.name.c_str(), index); + return std::pair(sanitizeFieldName(), index); }) }; return m.at(fieldName, -1); } @@ -200,7 +208,7 @@ constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value, F using UnwrappedMemberType = std::remove_reference_t; if constexpr (isReflectableClass()) { // nested data-structure const auto subfields = getNumberOfNonNullSubfields(getAnnotatedMember(unwrapPointer(fieldValue))); - FieldDescription auto field = newFieldHeader(buffer, member.name.c_str(), parent.hierarchyDepth + 1, FWD(fieldValue), subfields); + FieldDescription auto field = newFieldHeader(buffer, sanitizeFieldName(), parent.hierarchyDepth + 1, FWD(fieldValue), subfields); const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, START_MARKER_INST); const std::size_t posStartDataStart = buffer.size(); serialise(buffer, getAnnotatedMember(unwrapPointer(fieldValue)), field); // do not inspect annotation itself @@ -208,7 +216,7 @@ constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value, F updateSize(buffer, posSizePositionStart, posStartDataStart); return; } else { // field is a (possibly annotated) primitive type - FieldDescription auto field = newFieldHeader(buffer, member.name.c_str(), parent.hierarchyDepth + 1, fieldValue, 0); + FieldDescription auto field = newFieldHeader(buffer, sanitizeFieldName(), parent.hierarchyDepth + 1, fieldValue, 0); FieldHeaderWriter::template put(buffer, field, fieldValue); } } @@ -220,7 +228,7 @@ template constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value) { putHeaderInfo(buffer); const auto subfields = detail::getNumberOfNonNullSubfields(value); - auto field = detail::newFieldHeader(buffer, refl::reflect(value).name.c_str(), 0, value, subfields); + auto field = detail::newFieldHeader(buffer, sanitizeFieldName>().name.size, refl::reflect>().name>(), 0, value, subfields); const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, START_MARKER_INST); const std::size_t posStartDataStart = buffer.size(); detail::serialise(buffer, value, field); @@ -338,7 +346,7 @@ constexpr void deserialise(IoBuffer &buffer, ReflectableClass auto &value, Deser field.intDataType = IoSerialiser::getDataTypeId(); } else { constexpr int requestedType = IoSerialiser::getDataTypeId(); - if (requestedType != field.intDataType) { // mismatching data-type + if (requestedType != field.intDataType && requestedType != IoSerialiser::getDataTypeId()) { // mismatching data-type moveToFieldEndBufferPosition(buffer, field); if constexpr (check == ProtocolCheck::IGNORE) { return; // don't write -> skip to next @@ -387,7 +395,7 @@ constexpr void deserialise(IoBuffer &buffer, ReflectableClass auto &value, Deser if constexpr (isReflectableClass()) { buffer.set_position(field.headerStart); // reset buffer position for the nested deserialiser to read again field.hierarchyDepth++; - field.fieldName = member.name.c_str(); // N.B. needed since member.name is referring to compile-time const string + field.fieldName = sanitizeFieldName(); // N.B. needed since member.name is referring to compile-time const string deserialise(buffer, unwrapPointerCreateIfAbsent(member(value)), info, field); field.hierarchyDepth--; field.subfields = previousSubFields - 1; diff --git a/src/serialiser/include/IoSerialiserCmwLight.hpp b/src/serialiser/include/IoSerialiserCmwLight.hpp index ddd8286a..0809df63 100644 --- a/src/serialiser/include/IoSerialiserCmwLight.hpp +++ b/src/serialiser/include/IoSerialiserCmwLight.hpp @@ -5,8 +5,8 @@ #include #pragma clang diagnostic push -#pragma ide diagnostic ignored "cppcoreguidelines-avoid-magic-numbers" -#pragma ide diagnostic ignored "cppcoreguidelines-avoid-c-arrays" +#pragma ide diagnostic ignored "cppcoreguidelines-avoid-magic-numbers" +#pragma ide diagnostic ignored "cppcoreguidelines-avoid-c-arrays" namespace opencmw { @@ -18,7 +18,7 @@ namespace cmwlight { void skipString(IoBuffer &buffer); template -inline void skipArray(IoBuffer &buffer) { +void skipRawArray(IoBuffer &buffer) { if constexpr (is_stringlike) { for (auto i = buffer.get(); i > 0; i--) { skipString(buffer); @@ -28,24 +28,30 @@ inline void skipArray(IoBuffer &buffer) { } } -inline void skipString(IoBuffer &buffer) { skipArray(buffer); } +template +void skipArray(IoBuffer &buffer) { + buffer.skip(2 * static_cast(sizeof(int32_t))); + skipRawArray(buffer); +} + +inline void skipString(IoBuffer &buffer) { skipRawArray(buffer); } template -inline void skipMultiArray(IoBuffer &buffer) { +void skipMultiArray(IoBuffer &buffer) { buffer.skip(buffer.get() * static_cast(sizeof(int32_t))); // skip size header - skipArray(buffer); // skip elements + skipRawArray(buffer); // skip elements } template -inline int getTypeId() { +int getTypeId() { return IoSerialiser::getDataTypeId(); } template -inline int getTypeIdVector() { +int getTypeIdVector() { return IoSerialiser>::getDataTypeId(); } template -inline int getTypeIdMultiArray() { +int getTypeIdMultiArray() { return IoSerialiser>::getDataTypeId(); } @@ -53,12 +59,12 @@ inline int getTypeIdMultiArray() { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFF; } // default value + static constexpr uint8_t getDataTypeId() { return 0xFF; } // default value }; template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 0; } else if constexpr (std::is_same_v) { return 1; } @@ -70,6 +76,7 @@ struct IoSerialiser { else if constexpr (std::is_same_v) { return 201; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { @@ -83,7 +90,7 @@ struct IoSerialiser { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 7; } + static constexpr uint8_t getDataTypeId() { return 7; } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { buffer.put(value); @@ -96,9 +103,9 @@ struct IoSerialiser { template struct IoSerialiser { - using MemberType = typename T::value_type; + using MemberType = T::value_type; - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 9; } else if constexpr (std::is_same_v) { return 10; } @@ -111,20 +118,25 @@ struct IoSerialiser { else if constexpr (opencmw::is_stringlike ) { return 16; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { + buffer.put(1); + buffer.put(static_cast(value.size())); buffer.put(value); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, T &value) noexcept { - buffer.getArray(value); + const int nx = buffer.get(); + const int ny = buffer.get(); + buffer.getArray(value, nx * ny); } }; template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v && T::n_dims_ == 1) { return cmwlight::getTypeIdVector(); } else if constexpr (std::is_same_v && T::n_dims_ == 1) { return cmwlight::getTypeIdVector(); } @@ -155,6 +167,7 @@ struct IoSerialiser { else if constexpr (opencmw::is_stringlike ) { return 32; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { @@ -169,7 +182,7 @@ struct IoSerialiser { constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, T &value) noexcept { const std::array dimWire = buffer.getArray(); for (auto i = 0U; i < T::n_dims_; i++) { - value.dimensions()[i] = static_cast(dimWire[i]); + value.dimensions()[i] = static_cast(dimWire[i]); } value.element_count() = value.dimensions()[T::n_dims_ - 1]; value.stride(T::n_dims_ - 1) = 1; @@ -185,7 +198,7 @@ struct IoSerialiser { template struct IoSerialiser> { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 9; } else if constexpr (std::is_same_v) { return 10; } @@ -198,6 +211,7 @@ struct IoSerialiser> { else if constexpr (opencmw::is_stringlike ) { return 16; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const std::set &value) noexcept { @@ -232,22 +246,24 @@ struct IoSerialiser> { } } }; + template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFC; } + static constexpr uint8_t getDataTypeId() { return 0x08; } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const START_MARKER &/*value*/) noexcept { buffer.put(static_cast(field.subfields)); } - constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const START_MARKER &) { - // do not do anything, as the start marker is of size zero and only the type byte is important + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto &field, const START_MARKER &) { + field.subfields = static_cast(buffer.get()); + field.dataStartPosition = buffer.position(); } }; template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFE; } + static constexpr uint8_t getDataTypeId() { return 0xFE; } static void serialise(IoBuffer &/*buffer*/, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -260,7 +276,7 @@ struct IoSerialiser { template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFD; } + static constexpr uint8_t getDataTypeId() { return 0xFD; } static void serialise(IoBuffer &/*buffer*/, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -281,6 +297,7 @@ struct IoSerialiser { else if (typeId == getTypeId()) { buffer.skip(sizeof(char )); } else if (typeId == getTypeId()) { skipString(buffer); } // arrays + else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } @@ -290,6 +307,7 @@ struct IoSerialiser { else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } // 2D and Multi array + else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } @@ -318,7 +336,7 @@ struct FieldHeaderWriter { if (field.hierarchyDepth != 0) { buffer.put(field.fieldName); // full field name with zero termination } - if constexpr (!is_same_v) { // do not put startMarker type id into buffer + if (!is_same_v || field.hierarchyDepth != 0) { // do not put startMarker type id into buffer buffer.put(IoSerialiser::getDataTypeId()); // data type ID } IoSerialiser::serialise(buffer, field, getAnnotatedMember(unwrapPointer(data))); @@ -329,26 +347,27 @@ struct FieldHeaderWriter { template<> struct FieldHeaderReader { template - inline static void get(IoBuffer &buffer, DeserialiserInfo & /*info*/, FieldDescription auto &field) { + static void get(IoBuffer &buffer, DeserialiserInfo & /*info*/, FieldDescription auto &field) { field.headerStart = buffer.position(); field.dataEndPosition = std::numeric_limits::max(); - field.modifier = ExternalModifier::UNKNOWN; + field.modifier = UNKNOWN; if (field.subfields == 0) { field.intDataType = IoSerialiser::getDataTypeId(); - field.hierarchyDepth--; + --field.hierarchyDepth; field.dataStartPosition = buffer.position(); return; } if (field.subfields == -1) { - if (field.hierarchyDepth != 0) { // do not read field description for root element - field.fieldName = buffer.get(); // full field name + if (field.hierarchyDepth != 0) { // do not read field description for root element + field.fieldName = buffer.get(); // full field name + field.intDataType = buffer.get(); // data type ID + } else { + field.intDataType = IoSerialiser::getDataTypeId(); } - field.intDataType = IoSerialiser::getDataTypeId(); - field.subfields = static_cast(buffer.get()); } else { field.fieldName = buffer.get(); // full field name field.intDataType = buffer.get(); // data type ID - field.subfields--; // decrease the number of remaining fields in the structure... todo: adapt strategy for nested fields (has to somewhere store subfields) + --field.subfields; // decrease the number of remaining fields in the structure... } field.dataStartPosition = buffer.position(); } @@ -364,6 +383,111 @@ inline DeserialiserInfo checkHeaderInfo(IoBuffer &buffer, Deserialiser return info; } +/** + * The serialiser for std::variant is a bit special, as it contains a runtime-determined type, while in general the IoSerialiser assumes that the + * type can be deduced statically from the type via the getDataTypeId function. + * Therefore, for now only serialisation is implemented, and even there we have to retroactively overwrite the field header's type ID from within the + * serialise function. + */ +template +struct IoSerialiser> { + static constexpr uint8_t getDataTypeId() { + return IoSerialiser::getDataTypeId(); // this is just a stand-in and will be overwritten with the actual type id in the serialise function + } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::variant &value) noexcept { + std::visit([&buffer, &field](T &val) { + using StrippedT = std::remove_cvref_t; + // overwrite the field header with the actual type + buffer.resize(buffer.size() - sizeof(uint8_t)); + buffer.put(IoSerialiser::getDataTypeId()); + // serialise the contained value + IoSerialiser::serialise(buffer, field, val); + }, + value); + } + + constexpr static void deserialise(IoBuffer & buffer, FieldDescription auto const & parent, std::variant & value) { + auto deserialiseVariant = [&]() { + if (parent.intDataType == IoSerialiser::getDataTypeId()) { + T result; + IoSerialiser::deserialise(buffer, parent, result); + value = result; + return true; + } + return false; + }; + if (!(deserialiseVariant.template operator()() || ...)) { + throw ProtocolException(std::format("Type in data is not supported by the requested variant: {}", parent.intDataType)); + } + } +}; + +template +struct IoSerialiser> { + static constexpr uint8_t getDataTypeId() { + return IoSerialiser::getDataTypeId(); + } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &parentField, const std::map &value) noexcept { + buffer.put(static_cast(value.size())); + for (auto &[key, val] : value) { + if constexpr (isReflectableClass()) { // nested data-structure + const auto subfields = value.size(); + FieldDescription auto field = newFieldHeader(buffer, parentField.hierarchyDepth + 1, FWD(val), subfields); + [[maybe_unused]] const std::size_t posSizePositionStart = FieldHeaderWriter::put(buffer, field, val); + [[maybe_unused]] const std::size_t posStartDataStart = buffer.size(); + return; + } else { // field is a (possibly annotated) primitive type + FieldDescription auto field = detail::newFieldHeader(buffer, key.c_str(), parentField.hierarchyDepth + 1, val, 0); + FieldHeaderWriter::put(buffer, field, val); + } + } + } + + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto &parent, std::map &value) { + using protocol = CmwLight; + constexpr auto check = ProtocolCheck::IGNORE; + IoSerialiser::deserialise(buffer, parent, START_MARKER_INST); + auto field = detail::newFieldHeader(buffer, "", parent.hierarchyDepth, ValueType{}, parent.subfields); + while (buffer.position() < buffer.size()) { + DeserialiserInfo info; + auto previousSubFields = field.subfields; + FieldHeaderReader::get(buffer, info, field); + buffer.set_position(field.dataStartPosition); // skip to data start + + if (field.intDataType == IoSerialiser::getDataTypeId()) { // reached end of sub-structure + try { + IoSerialiser::deserialise(buffer, field, END_MARKER_INST); + } catch (...) { + if (detail::handleDeserialisationErrorAndSkipToNextField(buffer, field, info, "IoSerialiser<{}, END_MARKER>::deserialise(buffer, {}::{}, END_MARKER_INST): position {} vs. size {} -- exception: {}", + protocol::protocolName(), parent.fieldName, field.fieldName, buffer.position(), buffer.size(), what())) { + continue; + } + } + return; // step down to previous hierarchy depth + } + + const auto [fieldValue, _] = value.insert({ std::string{ field.fieldName }, ValueType{} }); + if constexpr (isReflectableClass()) { + field.intDataType = IoSerialiser::getDataTypeId(); + buffer.set_position(field.headerStart); // reset the buffer position for the nested deserialiser to read again + ++field.hierarchyDepth; + deserialise(buffer, fieldValue->second, info, field); + --field.hierarchyDepth; + field.subfields = previousSubFields - 1; + } else { + constexpr int requestedType = IoSerialiser::getDataTypeId(); + if (requestedType != field.intDataType && requestedType != IoSerialiser::getDataTypeId()) { // mismatching data-type + detail::moveToFieldEndBufferPosition(buffer, field); + detail::handleDeserialisationError(info, "mismatched field type for map field {} - requested type: {} (typeID: {}) got: {}", field.fieldName, typeName, requestedType, field.intDataType); + continue; + } + IoSerialiser::deserialise(buffer, field, fieldValue->second); + } + } + } +}; } // namespace opencmw #pragma clang diagnostic pop diff --git a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp index 4c670ad5..0d0f92bc 100644 --- a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp +++ b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp @@ -4,10 +4,10 @@ #include #include +#include #include #include -#include #include #include @@ -33,12 +33,71 @@ struct SimpleTestData { opencmw::MultiArray d{ { 1, 2, 3, 4, 5, 6 }, { 2, 3 } }; std::unique_ptr e = nullptr; std::set f{ "one", "two", "three" }; - bool operator==(const ioserialiser_cmwlight_test::SimpleTestData &other) const { // deep comparison function - return a == other.a && ab == other.ab && abc == other.abc && b == other.b && c == other.c && cd == other.cd && d == other.d && ((!e && !other.e) || *e == *(other.e)) && f == other.f; + std::map g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } }; + bool operator==(const SimpleTestData &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; + } +}; +struct SimpleTestDataMapNested { + int g1; + int g2; + int g3; + + bool operator==(const SimpleTestDataMapNested &o) const = default; + bool operator==(const std::map &o) const { + return g1 == o.at("g1") && g2 == o.at("g2") && g3 == o.at("g3"); + } +}; +struct SimpleTestDataMapAsNested { + int a = 1337; + float ab = 13.37f; + double abc = 42.23; + std::string b = "hello"; + std::array c{ 3, 2, 1 }; + std::vector cd{ 2.3, 3.4, 4.5, 5.6 }; + std::vector ce{ "hello", "world" }; + opencmw::MultiArray d{ { 1, 2, 3, 4, 5, 6 }, { 2, 3 } }; + std::unique_ptr e = nullptr; + std::set f{ "one", "two", "three" }; + SimpleTestDataMapNested g{ 1, 2, 3 }; + bool operator==(const SimpleTestDataMapAsNested &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; + } + bool operator==(const SimpleTestData &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; } }; } // namespace ioserialiser_cmwlight_test -ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestData, a, ab, abc, b, c, cd, ce, d, e, f) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestData, a, ab, abc, b, c, cd, ce, d, e, f, g) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMapAsNested, a, ab, abc, b, c, cd, ce, d, e, f, g) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMapNested, g1, g2, g3) + +// small utility function that prints the content of a string in the classic hexedit way with address, hexadecimal and ascii representations +static std::string hexview(const std::string_view value, const std::size_t bytesPerLine = 4) { + std::string result; + result.reserve(value.size() * 4); + std::string alpha; // temporarily store the ascii representation + alpha.reserve(8 * bytesPerLine); + std::size_t i = 0; + for (auto c : value) { + if (i % (bytesPerLine * 8) == 0) { + result.append(std::format("{0:#08x} - {0:04} | ", i)); // print address in hex and decimal + } + result.append(std::format("{:02x} ", c)); + alpha.append(std::format("{}", std::isprint(c) ? c : '.')); + if ((i + 1) % 8 == 0) { + result.append(" "); + alpha.append(" "); + } + if ((i + 1) % (bytesPerLine * 8) == 0) { + result.append(std::format(" {}\n", alpha)); + alpha.clear(); + } + i++; + } + result.append(std::format("{:{}} {}\n", "", 3 * (9 * bytesPerLine - alpha.size()), alpha)); + return result; +}; TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { using namespace opencmw; @@ -48,18 +107,19 @@ TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { debug::Timer timer("IoClassSerialiser basic syntax", 30); IoBuffer buffer; + IoBuffer bufferMap; std::cout << std::format("buffer size (before): {} bytes\n", buffer.size()); - SimpleTestData data{ + SimpleTestDataMapAsNested data{ .a = 30, .ab = 1.2f, .abc = 1.23, - .b = "abc", + .b = "abcde", .c = { 5, 4, 3 }, .cd = { 2.1, 4.2 }, .ce = { "hallo", "welt" }, .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, - .e = std::make_unique(SimpleTestData{ + .e = std::make_unique(SimpleTestDataMapAsNested{ .a = 40, .ab = 2.2f, .abc = 2.23, @@ -68,31 +128,73 @@ TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { .cd = { 3.1, 1.2 }, .ce = { "ei", "gude" }, .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, - .e = nullptr }), - .f = { "four", "five" } + .e = nullptr, + .f = { "un", "deux", "trois" }, + .g = { 6, 6, 6 } }), + .f = { "four", "five" }, + .g = { 4, 5, 6 } }; - // check that empty buffer cannot be deserialised - buffer.put(0L); - CHECK_THROWS_AS((opencmw::deserialise(buffer, data)), ProtocolException); - buffer.clear(); + SimpleTestData dataMap{ + .a = 30, + .ab = 1.2f, + .abc = 1.23, + .b = "abcde", + .c = { 5, 4, 3 }, + .cd = { 2.1, 4.2 }, + .ce = { "hallo", "welt" }, + .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, + .e = std::make_unique(SimpleTestData{ + .a = 40, + .ab = 2.2f, + .abc = 2.23, + .b = "abcdef", + .c = { 9, 8, 7 }, + .cd = { 3.1, 1.2 }, + .ce = { "ei", "gude" }, + .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, + .e = nullptr, + .f = { "un", "deux", "trois" }, + .g = { { "g1", 6 }, { "g2", 6 }, { "g3", 6 } } }), + .f = { "four", "five" }, + .g = { { "g1", 4 }, { "g2", 5 }, { "g3", 6 } } + }; - SimpleTestData data2; + SimpleTestDataMapAsNested data2; REQUIRE(data != data2); + SimpleTestDataMapAsNested dataMap2; + REQUIRE(dataMap != dataMap2); std::cout << "object (short): " << ClassInfoShort << data << '\n'; std::cout << std::format("object (std::format): {}\n", data); std::cout << "object (long): " << ClassInfoVerbose << data << '\n'; - opencmw::serialise(buffer, data); + opencmw::serialise(buffer, data); + opencmw::serialise(bufferMap, dataMap); std::cout << std::format("buffer size (after): {} bytes\n", buffer.size()); - buffer.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly + buffer.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly + bufferMap.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly buffer.reset(); + bufferMap.reset(); + + std::cout << "buffer contentsSubObject: \n" + << hexview(buffer.asString()) << "\n"; + std::cout << "buffer contentsMap: \n" + << hexview(bufferMap.asString()) << "\n"; + + REQUIRE(buffer.asString() == bufferMap.asString()); + + // TODO: fix this case auto result = opencmw::deserialise(buffer, data2); std::cout << "deserialised object (long): " << ClassInfoVerbose << data2 << '\n'; std::cout << "deserialisation messages: " << result << std::endl; REQUIRE(data == data2); + + auto result2 = opencmw::deserialise(bufferMap, dataMap2); + std::cout << "deserialised object (long): " << ClassInfoVerbose << dataMap2 << '\n'; + std::cout << "deserialisation messages: " << result2 << std::endl; + REQUIRE(dataMap == dataMap2); } REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred debug::resetStats(); @@ -120,9 +222,10 @@ struct SimpleTestDataMoreFields { bool operator==(const SimpleTestDataMoreFields &) const = default; std::unique_ptr e = nullptr; std::set f{ "one", "two", "three" }; + std::map g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } }; }; } // namespace ioserialiser_cmwlight_test -ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMoreFields, a2, ab2, abc2, b2, c2, cd2, ce2, d2, e2, a, ab, abc, b, c, cd, ce, d, e, f) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMoreFields, a2, ab2, abc2, b2, c2, cd2, ce2, d2, e2, a, ab, abc, b, c, cd, ce, d, e, f, g) #pragma clang diagnostic pop TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { @@ -144,30 +247,324 @@ TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { .cd = { 2.1, 4.2 }, .ce = { "hallo", "welt" }, .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, - .f = { "four", "six" } + .f = { "four", "six" }, + .g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } } }; SimpleTestDataMoreFields data2; std::cout << std::format("object (std::format): {}\n", data); - opencmw::serialise(buffer, data); + opencmw::serialise(buffer, data); buffer.reset(); - auto result = opencmw::deserialise(buffer, data2); + auto result = opencmw::deserialise(buffer, data2); std::cout << std::format("deserialised object (std::format): {}\n", data2); std::cout << "deserialisation messages: " << result << std::endl; - REQUIRE(result.setFields["root"] == std::vector{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 }); + REQUIRE(result.setFields["root"] == std::vector{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 }); REQUIRE(result.additionalFields.empty()); REQUIRE(result.exceptions.empty()); std::cout << "\nand now the other way round!\n\n"; buffer.clear(); - opencmw::serialise(buffer, data2); + opencmw::serialise(buffer, data2); buffer.reset(); - auto result_back = opencmw::deserialise(buffer, data); + auto result_back = opencmw::deserialise(buffer, data); std::cout << std::format("deserialised object (std::format): {}\n", data); std::cout << "deserialisation messages: " << result_back << std::endl; - REQUIRE(result_back.setFields["root"] == std::vector{ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 }); + REQUIRE(result_back.setFields["root"] == std::vector{ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 }); REQUIRE(result_back.additionalFields.size() == 8); REQUIRE(result_back.exceptions.size() == 8); } REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred debug::resetStats(); } + +namespace ioserialiser_cmwlight_test { +struct IntegerMap { + int x_8 = 1336; // fieldname gets mapped to "8" + int foo = 42; + int bar = 45; +}; +} // namespace ioserialiser_cmwlight_test +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::IntegerMap, x_8, foo, bar) + +TEST_CASE("IoClassSerialiserCmwLight deserialise into map", "[IoClassSerialiser]") { + using namespace opencmw; + using namespace ioserialiser_cmwlight_test; + debug::resetStats(); + { + // serialise + IoBuffer buffer; + IntegerMap input{ 23, 13, 37 }; + opencmw::serialise(buffer, input); + buffer.reset(); + REQUIRE(buffer.size() == sizeof(int32_t) /* map size */ + refl::reflect(input).members.size /* map entries */ * (sizeof(int32_t) /* string lengths */ + sizeof(uint8_t) /* type */ + sizeof(int32_t) /* int */) + 2 + 4 + 4 /* strings + \0 */); + // std::cout << hexview(buffer.asString()); + + // deserialise + std::map deserialised{}; + DeserialiserInfo info; + auto field = opencmw::detail::newFieldHeader(buffer, "map", 0, deserialised, -1); + FieldHeaderReader::get(buffer, info, field); + IoSerialiser>::deserialise(buffer, field, deserialised); + + // check for correctness + REQUIRE(deserialised.size() == 3); + REQUIRE(deserialised["8"] == 23); + REQUIRE(deserialised["foo"] == 13); + REQUIRE(deserialised["bar"] == 37); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} + +TEST_CASE("IoClassSerialiserCmwLight deserialise variant map", "[IoClassSerialiser]") { + using namespace opencmw; + using namespace ioserialiser_cmwlight_test; + debug::resetStats(); + { + const std::string_view expected{ "\x03\x00\x00\x00" // 3 fields + "\x02\x00\x00\x00" + "a\x00" + "\x03" + "\x23\x00\x00\x00" // "a" -> int:0x23 + "\x02\x00\x00\x00" + "b\x00" + "\x06" + "\xEC\x51\xB8\x1E\x85\xEB\xF5\x3F" // "b" -> double:1.337 + "\x02\x00\x00\x00" + "c\x00" + "\x07" + "\x04\x00\x00\x00" + "foo\x00" // "c" -> "foo" + , + 45 }; + std::map> map{ { "a", 0x23 }, { "b", 1.37 }, { "c", "foo" } }; + + IoBuffer buffer; + auto field = opencmw::detail::newFieldHeader(buffer, "map", 0, map, -1); + IoSerialiser>>::serialise(buffer, field, map); + buffer.reset(); + + // std::print("expected:\n{}\ngot:\n{}\n", hexview(expected), hexview(buffer.asString())); + REQUIRE(buffer.asString() == expected); + + std::map> mapDeserialised{}; + auto fieldToDeserialise = opencmw::detail::newFieldHeader(buffer, "map", 0, map, -1); + IoSerialiser>>::deserialise(buffer, fieldToDeserialise, mapDeserialised); + + //std::println("mapDeserialised: {}", mapDeserialised | std::views::keys); + REQUIRE(mapDeserialised.size() == 3); + REQUIRE(mapDeserialised == map); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} + +namespace opencmw::serialiser::cmwlighttests { +struct CmwLightHeaderOptions { + int64_t b; // SOURCE_ID + std::map e; + // can potentially contain more and arbitrary data + // accessors to make code more readable + int64_t &sourceId() { return b; } + std::map sessionBody; +}; +struct CmwLightHeader { + int8_t x_2; // REQ_TYPE_TAG + int64_t x_0; // ID_TAG + std::string x_1; // DEVICE_NAME + std::string f; // PROPERTY_NAME + int8_t x_7; // UPDATE_TYPE + std::string d; // SESSION_ID + std::unique_ptr x_3; + // accessors to make code more readable + int8_t &requestType() { return x_2; } + int64_t &id() { return x_0; } + std::string &device() { return x_1; } + std::string &property() { return f; } + int8_t &updateType() { return x_7; } + std::string &sessionId() { return d; } + std::unique_ptr &options() { return x_3; } +}; +struct DigitizerVersion { + std::string classVersion; + std::string deployUnitVersion; + std::string fesaVersion; + std::string gr_flowgraph_version; + std::string gr_digitizer_version; + std::string daqAPIVersion; +}; +struct Empty {}; +struct StatusProperty { + int32_t control; + std::vector detailedStatus; + std::vector detailedStatus_labels; + std::vector detailedStatus_severity; + std::vector error_codes; + std::vector error_cycle_names; + std::vector error_messages; + std::vector error_timestamps; + bool interlock; + bool modulesReady; + bool opReady; + int32_t powerState; + int32_t status; +}; +} // namespace opencmw::serialiser::cmwlighttests +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::CmwLightHeaderOptions, b, e) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::CmwLightHeader, x_2, x_0, x_1, f, x_7, d, x_3) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::DigitizerVersion, classVersion, deployUnitVersion, fesaVersion, gr_flowgraph_version, gr_digitizer_version, daqAPIVersion) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::StatusProperty, control, detailedStatus, detailedStatus_labels, detailedStatus_severity, error_codes, error_cycle_names, error_messages, error_timestamps, interlock, modulesReady, opReady, powerState, status) +REFL_TYPE(opencmw::serialiser::cmwlighttests::Empty) +REFL_END + +TEST_CASE("IoClassSerialiserCmwLight Deserialise rda3 data", "[IoClassSerialiser]") { + // ensure that important rda3 messages can be properly deserialized + using namespace opencmw; + debug::resetStats(); + using namespace opencmw::serialiser::cmwlighttests; + { + std::vector rda3ConnectReply = { // + /* 0 */ 0x07, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, /**/ 0x30, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ 0....... + /* 16 */ 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x31, /**/ 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x00, 0x02, // .......1 ........ + /* 32 */ 0x00, 0x00, 0x00, 0x32, 0x00, 0x01, 0x03, 0x02, /**/ 0x00, 0x00, 0x00, 0x33, 0x00, 0x08, 0x02, 0x00, // ...2.... ...3.... + /* 48 */ 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x62, 0x00, /**/ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ......b. ........ + /* 64 */ 0x00, 0x02, 0x00, 0x00, 0x00, 0x65, 0x00, 0x08, /**/ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // .....e.. ........ + /* 80 */ 0x37, 0x00, 0x01, 0x70, 0x02, 0x00, 0x00, 0x00, /**/ 0x64, 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x00, // 7..j.... d....... + /* 96 */ 0x02, 0x00, 0x00, 0x00, 0x66, 0x00, 0x07, 0x01, /**/ 0x00, 0x00, 0x00, 0x00 }; // ...f... .... + IoBuffer buffer{ rda3ConnectReply.data(), rda3ConnectReply.size() }; + CmwLightHeader deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 7); + + REQUIRE(deserialised.requestType() == 3); + REQUIRE(deserialised.id() == 0); + REQUIRE(deserialised.device().empty()); + REQUIRE(deserialised.property().empty()); + REQUIRE(deserialised.updateType() == 0x70); + REQUIRE(deserialised.sessionId().empty()); + REQUIRE(deserialised.options()->sourceId() == 0); + REQUIRE(deserialised.options()->sessionBody.empty()); + } + { + // reply req type: session confirm + std::vector data = { /* */ 0x07, 0x00, 0x00, 0x00, // .... + /* 4 */ 0x02, 0x00, 0x00, 0x00, 0x30, 0x00, 0x04, 0x09, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, // ....0... ........ + /* 20 */ 0x00, 0x00, 0x00, 0x31, 0x00, 0x07, 0x08, 0x00, /**/ 0x00, 0x00, 0x47, 0x47, 0x47, 0x47, 0x30, 0x30, // ...1.... ..GGGG00 + /* 36 */ 0x32, 0x00, 0x02, 0x00, 0x00, 0x00, 0x32, 0x00, /**/ 0x01, 0x0b, 0x02, 0x00, 0x00, 0x00, 0x33, 0x00, // 2.....2. ......3. + /* 52 */ 0x08, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x65, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, // ........ .e...... + /* 68 */ 0x02, 0x00, 0x00, 0x00, 0x37, 0x00, 0x01, 0x00, /**/ 0x02, 0x00, 0x00, 0x00, 0x64, 0x00, 0x07, 0x25, // ....7... ....d..% + /* 84 */ 0x01, 0x00, 0x00, 0x52, 0x65, 0x6d, 0x6f, 0x74, /**/ 0x65, 0x48, 0x6f, 0x73, 0x74, 0x49, 0x6e, 0x66, // ...Remot eHostInf + /* 100 */ 0x6f, 0x49, 0x6d, 0x70, 0x6c, 0x5b, 0x6e, 0x61, /**/ 0x6d, 0x65, 0x3d, 0x66, 0x65, 0x73, 0x61, 0x2d, // oImpl[na me=fesa- + /* 116 */ 0x65, 0x78, 0x70, 0x6c, 0x6f, 0x72, 0x65, 0x72, /**/ 0x2d, 0x61, 0x70, 0x70, 0x3b, 0x20, 0x75, 0x73, // explorer -app; us + /* 132 */ 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x61, /**/ 0x61, 0x61, 0x61, 0x61, 0x61, 0x3b, 0x20, 0x61, // erName=a aaaaa; a + /* 148 */ 0x70, 0x70, 0x49, 0x64, 0x3d, 0x5b, 0x61, 0x70, /**/ 0x70, 0x3d, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, // ppId=[ap p=ffffff + /* 164 */ 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x26, /**/ 0x66, 0x66, 0x66, 0x3b, 0x76, 0x65, 0x72, 0x3d, // fffffff- fff;ver= + /* 180 */ 0x31, 0x39, 0x2e, 0x30, 0x2e, 0x30, 0x3b, 0x75, /**/ 0x69, 0x64, 0x3d, 0x61, 0x61, 0x61, 0x61, 0x61, // 19.0.0;u id=aaaaa + /* 196 */ 0x61, 0x3b, 0x68, 0x6f, 0x73, 0x74, 0x3d, 0x53, /**/ 0x53, 0x53, 0x53, 0x53, 0x53, 0x53, 0x53, 0x3b, // a;host=S SSSSSSS; + /* 212 */ 0x70, 0x69, 0x64, 0x3d, 0x31, 0x39, 0x31, 0x36, /**/ 0x31, 0x36, 0x3b, 0x5d, 0x3b, 0x20, 0x70, 0x72, // pid=1916 16;]; pr + /* 228 */ 0x6f, 0x63, 0x65, 0x73, 0x73, 0x3d, 0x66, 0x65, /**/ 0x73, 0x61, 0x2d, 0x65, 0x78, 0x70, 0x6c, 0x6f, // ocess=fe sa-explo + /* 244 */ 0x72, 0x65, 0x72, 0x2d, 0x61, 0x70, 0x70, 0x3b, /**/ 0x20, 0x70, 0x69, 0x64, 0x3d, 0x31, 0x39, 0x31, // rer-app; pid=191 + /* 260 */ 0x36, 0x31, 0x36, 0x3b, 0x20, 0x61, 0x64, 0x64, /**/ 0x72, 0x65, 0x73, 0x73, 0x3d, 0x74, 0x63, 0x70, // 616; add ress=tcp + /* 276 */ 0x3a, 0x2f, 0x2f, 0x53, 0x53, 0x53, 0x53, 0x53, /**/ 0x53, 0x53, 0x53, 0x3a, 0x30, 0x3b, 0x20, 0x73, // ://SSSSS SSS:0; s + /* 292 */ 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, /**/ 0x3d, 0x32, 0x30, 0x32, 0x34, 0x2d, 0x30, 0x37, // tartTime =2024-07 + /* 308 */ 0x2d, 0x30, 0x34, 0x20, 0x31, 0x31, 0x3a, 0x31, /**/ 0x31, 0x3a, 0x31, 0x32, 0x3b, 0x20, 0x63, 0x6f, // -04 11:1 1:12; co + /* 324 */ 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, /**/ 0x54, 0x69, 0x6d, 0x65, 0x3d, 0x41, 0x62, 0x6f, // nnection Time=Abo + /* 340 */ 0x75, 0x74, 0x20, 0x61, 0x67, 0x6f, 0x3b, 0x20, /**/ 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3d, // ut ago; version= + /* 356 */ 0x31, 0x30, 0x2e, 0x33, 0x2e, 0x30, 0x3b, 0x20, /**/ 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, // 10.3.0; language + /* 372 */ 0x3d, 0x4a, 0x61, 0x76, 0x61, 0x5d, 0x31, 0x00, /**/ 0x02, 0x00, 0x00, 0x00, 0x66, 0x00, 0x07, 0x08, // =Java]1. ....f... + /* 388 */ 0x00, 0x00, 0x00, 0x56, 0x65, 0x72, 0x73, 0x69, /**/ 0x6f, 0x6e, 0x00 }; //...Versi on. + IoBuffer buffer{ data.data(), data.size() }; + CmwLightHeader deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 7); + } + { + // Reply with Req Type = Reply, gets sent after get request + std::vector data = { /* */ 0x06, 0x00, // .. + /* 2 */ 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x63, 0x6c, /**/ 0x61, 0x73, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, // ......cl assVersi + /* 18 */ 0x6f, 0x6e, 0x00, 0x07, 0x06, 0x00, 0x00, 0x00, /**/ 0x36, 0x2e, 0x30, 0x2e, 0x30, 0x00, 0x0e, 0x00, // on...... 6.0.0... + /* 34 */ 0x00, 0x00, 0x64, 0x61, 0x71, 0x41, 0x50, 0x49, /**/ 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, // ..daqAPI Version. + /* 50 */ 0x07, 0x04, 0x00, 0x00, 0x00, 0x32, 0x2e, 0x30, /**/ 0x00, 0x12, 0x00, 0x00, 0x00, 0x64, 0x65, 0x70, // .....2.0 .....dep + /* 66 */ 0x6c, 0x6f, 0x79, 0x55, 0x6e, 0x69, 0x74, 0x56, /**/ 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x07, // loyUnitV ersion.. + /* 82 */ 0x06, 0x00, 0x00, 0x00, 0x36, 0x2e, 0x30, 0x2e, /**/ 0x30, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x66, 0x65, // ....6.0. 0.....fe + /* 98 */ 0x73, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, /**/ 0x6e, 0x00, 0x07, 0x06, 0x00, 0x00, 0x00, 0x37, // saVersio n......7 + /* 114 */ 0x2e, 0x33, 0x2e, 0x30, 0x00, 0x15, 0x00, 0x00, /**/ 0x00, 0x67, 0x72, 0x5f, 0x64, 0x69, 0x67, 0x69, // .3.0.... .gr_digi + /* 130 */ 0x74, 0x69, 0x7a, 0x65, 0x72, 0x5f, 0x76, 0x65, /**/ 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x07, 0x08, // tizer_ve rsion... + /* 146 */ 0x00, 0x00, 0x00, 0x35, 0x2e, 0x31, 0x2e, 0x34, /**/ 0x2e, 0x30, 0x00, 0x15, 0x00, 0x00, 0x00, 0x67, // ...5.1.4 .0.....g + /* 162 */ 0x72, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x67, 0x72, /**/ 0x61, 0x70, 0x68, 0x5f, 0x76, 0x65, 0x72, 0x73, // r_flowgr aph_vers + /* 178 */ 0x69, 0x6f, 0x6e, 0x00, 0x07, 0x08, 0x00, 0x00, /**/ 0x00, 0x35, 0x2e, 0x30, 0x2e, 0x32, 0x2e, 0x30, // ion..... .5.0.2.0 + /* 194 */ 0x00, 0x01, 0x62, 0x03, 0x00, 0x00, 0x00, 0x02, /**/ 0x00, 0x00, 0x00, 0x35, 0x00, 0x04, 0x88, 0x39, // ..b..... ...5...9 + /* 210 */ 0xfe, 0x41, 0x88, 0xf7, 0xde, 0x17, 0x02, 0x00, /**/ 0x00, 0x00, 0x36, 0x00, 0x04, 0x00, 0x00, 0x00, // .A...... ..6..... + /* 226 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x78, 0x00, 0x08, 0x03, 0x00, 0x00, 0x00, // ........ .x...... + /* 242 */ 0x09, 0x00, 0x00, 0x00, 0x61, 0x63, 0x71, 0x53, /**/ 0x74, 0x61, 0x6d, 0x70, 0x00, 0x04, 0x88, 0x39, // ....acqS tamp...9 + /* 258 */ 0xfe, 0x41, 0x88, 0xf7, 0xde, 0x17, 0x05, 0x00, /**/ 0x00, 0x00, 0x74, 0x79, 0x70, 0x65, 0x00, 0x03, // .A...... ..type.. + /* 274 */ 0x02, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, /**/ 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, // ........ version. + /* 280 */ 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03} ; // ....... + IoBuffer buffer{ data.data(), data.size() }; + DigitizerVersion deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 6); + } + { + std::vector data{ // + /* 0 */ 0x0d, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, /**/ 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x00, /**/ 0x03, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, /**/ 0x00, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, // ........ control. ........ .detaile + /* 32 */ 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x00, /**/ 0x09, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x01, 0x16, /**/ 0x00, 0x00, 0x00, 0x64, 0x65, 0x74, 0x61, 0x69, // dStatus. ........ ........ ...detai + /* 64 */ 0x6c, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, /**/ 0x73, 0x5f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, /**/ 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, /**/ 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x0f, 0x00, // ledStatu s_labels ........ ........ + /* 96 */ 0x00, 0x00, 0x6d, 0x79, 0x53, 0x74, 0x61, 0x74, /**/ 0x75, 0x73, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x31, /**/ 0x00, 0x0f, 0x00, 0x00, 0x00, 0x6d, 0x79, 0x53, /**/ 0x74, 0x61, 0x74, 0x75, 0x73, 0x4c, 0x61, 0x62, // ..myStat usLabel1 .....myS tatusLab + /* 128 */ 0x65, 0x6c, 0x32, 0x00, 0x18, 0x00, 0x00, 0x00, /**/ 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, /**/ 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x73, /**/ 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x00, // el2..... detailed Status_s everity. + /* 160 */ 0x0c, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, // ........ ........ ........ .error_c + /* 192 */ 0x6f, 0x64, 0x65, 0x73, 0x00, 0x0c, 0x01, 0x00, /**/ 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x10, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // odes.... ........ ........ ........ + /* 224 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + /* 256 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x65, 0x72, /**/ 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x79, 0x63, 0x6c, // ........ ........ ......er ror_cycl + /* 288 */ 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x00, /**/ 0x10, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, /**/ 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, // e_names. ........ ........ ........ + /* 320 */ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, // ........ ........ ........ ........ + /* 352 */ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + /* 384 */ 0x01, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, /**/ 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x00, /**/ 0x10, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, // ........ .error_m essages. ........ + /* 416 */ 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, // ........ ........ ........ ........ + /* 448 */ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + /* 480 */ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x74, // ........ ........ ........ .error_t + /* 512 */ 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, /**/ 0x73, 0x00, 0x0d, 0x01, 0x00, 0x00, 0x00, 0x10, /**/ 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // imestamp s....... ........ ........ + /* 544 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + /* 576 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + /* 608 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + /* 640 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, /**/ 0x00, 0x00, 0x00, 0x69, 0x6e, 0x74, 0x65, 0x72, // ........ ........ ........ ...inter + /* 672 */ 0x6c, 0x6f, 0x63, 0x6b, 0x00, 0x00, 0x00, 0x0d, /**/ 0x00, 0x00, 0x00, 0x6d, 0x6f, 0x64, 0x75, 0x6c, /**/ 0x65, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x00, /**/ 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x6f, 0x70, // lock.... ...modul esReady. ......op + /* 704 */ 0x52, 0x65, 0x61, 0x64, 0x79, 0x00, 0x00, 0x01, /**/ 0x0b, 0x00, 0x00, 0x00, 0x70, 0x6f, 0x77, 0x65, /**/ 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x00, 0x03, /**/ 0x01, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, // Ready... ....powe rState.. ........ + /* 736 */ 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x00, 0x03, /**/ 0x01, 0x00, 0x00, 0x00}; // status.. .... + IoBuffer buffer{ data.data(), data.size() }; + REQUIRE(buffer.size() == 748); + // deserialise once into an empty struct to verify that the fields can be correctly skipped and end up in the deserialiserInfo + Empty empty; + auto result = opencmw::deserialise(buffer, empty); + REQUIRE(result.additionalFields.size() == 13); // [root::control, root::detailedStatus, root::detailedStatus_labels, root::detailedStatus_severity, root::error_codes, root::error_cycle_names, root::error_messages, root::error_timestamps, root::interlock, root::modulesReady, root::opReady, root::powerState, root::status] [3, 9, 16, 12, 12, 16, 16, 13, 0, 0, 0, 3, 3] + REQUIRE(result.exceptions.size() == 13); // each missing field also produces an excception + REQUIRE(result.setFields["root"].empty()); + // deserialise into the correct domain object + buffer.reset(); + StatusProperty status; + auto result2 = opencmw::deserialise(buffer, status); + REQUIRE(result2.additionalFields.empty()); + REQUIRE(result2.exceptions.empty()); + REQUIRE(result2.setFields["root"sv].size() == 13); + REQUIRE(status.control == 0); + REQUIRE(status.detailedStatus == std::vector{true, true}); + REQUIRE(status.detailedStatus_labels == std::vector{"myStatusLabel1"s, "myStatusLabel2"s}); + REQUIRE(status.detailedStatus_severity == std::vector{0, 0}); + REQUIRE(status.error_codes.size() == 16); + REQUIRE(status.error_cycle_names.size() == 16); + REQUIRE(status.error_messages.size() == 16); + REQUIRE(status.error_timestamps.size() == 16); + REQUIRE_FALSE(status.interlock); + REQUIRE(status.modulesReady); + REQUIRE(status.opReady); + REQUIRE(status.powerState == 1); + REQUIRE(status.status == 1); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} From 7b85d35e085a4c069ff0e44a31a47522b1dedf0a Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Wed, 18 Feb 2026 16:48:27 +0100 Subject: [PATCH 4/5] JsonSerialiser: allow serialising list of objects Allows (de)serialising lists of reflectable objects. Due to the current structure of the code, some error reporting features are not supported for nested objects. Also fixes an off by one error which consumes one additional byte from the buffer after the object. Signed-off-by: Alexander Krimm --- src/serialiser/include/IoSerialiserJson.hpp | 132 ++++++++++-------- .../test/IoSerialiserJson_tests.cpp | 3 +- 2 files changed, 75 insertions(+), 60 deletions(-) diff --git a/src/serialiser/include/IoSerialiserJson.hpp b/src/serialiser/include/IoSerialiserJson.hpp index b546c51c..a99220b4 100644 --- a/src/serialiser/include/IoSerialiserJson.hpp +++ b/src/serialiser/include/IoSerialiserJson.hpp @@ -15,10 +15,10 @@ struct Json : Protocol<"Json"> {}; // as specified in https://www.json.org/json- namespace json { -constexpr inline bool isNumberChar(uint8_t c) { return (c >= '+' && c <= '9' && c != '/' && c != ',') || c == 'e' || c == 'E'; } -constexpr inline bool isWhitespace(uint8_t c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } -constexpr inline bool isHexNumberChar(int8_t c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -constexpr inline bool isHexNumber(const char *start, const size_t size) noexcept { +constexpr bool isNumberChar(uint8_t c) { return (c >= '+' && c <= '9' && c != '/' && c != ',') || c == 'e' || c == 'E'; } +constexpr bool isWhitespace(uint8_t c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } +constexpr bool isHexNumberChar(int8_t c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } +constexpr bool isHexNumber(const char *start, const size_t size) noexcept { for (size_t i = 0; i < size; i++) { if (!isHexNumberChar(start[i])) { return false; @@ -28,7 +28,7 @@ constexpr inline bool isHexNumber(const char *start, const size_t size) noexcept } template -inline void writeString(IoBuffer &buffer, const T &string) { + void writeString(IoBuffer &buffer, const T &string) { buffer.put('"'); for (const char c : string) { switch (c) { @@ -135,19 +135,19 @@ inline std::string_view readKey(IoBuffer &buffer) { return buffer.asString(start, static_cast(i - start)); } -inline constexpr void consumeWhitespace(IoBuffer &buffer) { + constexpr void consumeWhitespace(IoBuffer &buffer) { while (buffer.position() < buffer.size() && isWhitespace(buffer.at(buffer.position()))) { buffer.skip(1); } } template -inline constexpr void assignArray(std::vector &value, const std::vector &result) { + constexpr void assignArray(std::vector &value, const std::vector &result) { value = result; } template -inline constexpr void assignArray(std::array &value, const std::vector &result) { + constexpr void assignArray(std::array &value, const std::vector &result) { if (N != result.size()) { throw ProtocolException("vector -> array size mismatch: source<{}>[{}]={} vs. destination std::array", typeName, result.size(), result, typeName, N); } @@ -156,9 +156,9 @@ inline constexpr void assignArray(std::array &value, const std::vector } } -inline constexpr void skipValue(IoBuffer &buffer); + constexpr void skipValue(IoBuffer &buffer); -inline void skipField(IoBuffer &buffer) { +inline void skipField(IoBuffer &buffer) { consumeWhitespace(buffer); std::ignore = readString(buffer); consumeWhitespace(buffer); @@ -170,7 +170,7 @@ inline void skipField(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipObject(IoBuffer &buffer) { + constexpr void skipObject(IoBuffer &buffer) { if (buffer.get() != '{') { throw ProtocolException("error: expected {"); } @@ -187,13 +187,13 @@ inline constexpr void skipObject(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipNumber(IoBuffer &buffer) { + constexpr void skipNumber(IoBuffer &buffer) { while (isNumberChar(buffer.get())) { } buffer.skip(-1); } -inline constexpr void skipArray(IoBuffer &buffer) { + constexpr void skipArray(IoBuffer &buffer) { if (buffer.get() != '[') { throw ProtocolException("error: expected ["); } @@ -211,13 +211,13 @@ inline constexpr void skipArray(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipValue(IoBuffer &buffer) { + constexpr void skipValue(IoBuffer &buffer) { consumeWhitespace(buffer); const auto firstChar = buffer.at(buffer.position()); if (firstChar == '{') { - json::skipObject(buffer); + skipObject(buffer); } else if (firstChar == '[') { - json::skipArray(buffer); + skipArray(buffer); } else if (firstChar == '"') { std::ignore = readString(buffer); } else if (buffer.size() - buffer.position() > 5 && buffer.asString(buffer.position(), 5) == "false") { @@ -227,17 +227,17 @@ inline constexpr void skipValue(IoBuffer &buffer) { } else if (buffer.size() - buffer.position() > 4 && buffer.asString(buffer.position(), 4) == "null") { buffer.skip(4); } else { // skip number - json::skipNumber(buffer); + skipNumber(buffer); } } } // namespace json template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 1; } - constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { + static constexpr uint8_t getDataTypeId() { return 1; } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { using namespace std::string_view_literals; - buffer.put("}"sv); + buffer.put("}"sv); } constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const END_MARKER &) { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -246,10 +246,10 @@ struct IoSerialiser { template<> struct IoSerialiser { // catch all template - inline static constexpr uint8_t getDataTypeId() { return 2; } - constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const START_MARKER &/*value*/) noexcept { + static constexpr uint8_t getDataTypeId() { return 2; } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const START_MARKER &/*value*/) noexcept { using namespace std::string_view_literals; - buffer.put("{"sv); + buffer.put("{"sv); } constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const START_MARKER & /*value*/) { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -258,8 +258,8 @@ struct IoSerialiser { // catch all template template<> struct IoSerialiser { // because json does not explicitly provide the datatype, all types except nested classes provide data type OTHER - inline static constexpr uint8_t getDataTypeId() { return 0; } - constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const OTHER &) { + static constexpr uint8_t getDataTypeId() { return 0; } + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const OTHER &) { json::skipValue(buffer); } }; @@ -269,7 +269,7 @@ struct FieldHeaderWriter { template constexpr std::size_t static put(IoBuffer &buffer, FieldDescription auto &&field, const DataType &data) { using namespace std::string_view_literals; - constexpr auto WITHOUT = opencmw::IoBuffer::MetaInfo::WITHOUT; + constexpr auto WITHOUT = IoBuffer::MetaInfo::WITHOUT; if constexpr (std::is_same_v) { if (field.fieldName.size() > 0 && field.hierarchyDepth > 0) { buffer.put("\""sv); @@ -280,7 +280,7 @@ struct FieldHeaderWriter { return 0; } if constexpr (std::is_same_v) { - if (buffer.template at(buffer.size() - 2) == ',') { + if (buffer.at(buffer.size() - 2) == ',') { // proceeded by value, remove trailing comma buffer.resize(buffer.size() - 2); } @@ -322,7 +322,7 @@ struct FieldHeaderReader { if (buffer.at(buffer.position()) == '}') { // end marker buffer.skip(1); result.intDataType = IoSerialiser::getDataTypeId(); - result.dataStartPosition = buffer.position() + 1; + result.dataStartPosition = buffer.position(); result.dataEndPosition = std::numeric_limits::max(); // not defined for non-skippable data // set rest of fields return; @@ -355,10 +355,10 @@ struct FieldHeaderReader { template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const bool &value) noexcept { using namespace std::string_view_literals; - buffer.put(value ? "true"sv : "false"sv); + buffer.put(value ? "true"sv : "false"sv); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, bool &value) { using namespace std::string_view_literals; @@ -379,7 +379,7 @@ struct IoSerialiser { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) { buffer.reserve_spare(30); // just reserve some spare capacity and expect that all numbers are shorter const auto start = buffer.size(); @@ -432,7 +432,7 @@ struct IoSerialiser { template struct IoSerialiser { // catch all template - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { json::writeString(buffer, value); } @@ -444,9 +444,10 @@ struct IoSerialiser { // catch all template template struct IoSerialiser { // todo: arrays of objects - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } - inline constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &values) noexcept { - using MemberType = typename T::value_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &values) noexcept { + using MemberType = T::value_type; buffer.put('['); bool first = true; for (auto value : values) { @@ -460,7 +461,7 @@ struct IoSerialiser { buffer.put(']'); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { - using MemberType = typename T::value_type; + using MemberType = T::value_type; if (buffer.get() != '[') { throw ProtocolException("expected ["); } @@ -473,7 +474,7 @@ struct IoSerialiser { IoSerialiser::deserialise(buffer, field, entry); result.push_back(entry); json::consumeWhitespace(buffer); - const auto next = buffer.template get(); + const auto next = buffer.get(); if (next == ']') { break; } @@ -489,35 +490,35 @@ struct IoSerialiser { template struct IoSerialiser { - using V = typename T::value_type; - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER + using V = T::value_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &value) noexcept { using namespace std::string_view_literals; - buffer.put("{\n"sv); + buffer.put("{\n"sv); std::array dims; for (uint32_t i = 0U; i < T::n_dims_; i++) { dims[i] = static_cast(value.dimensions()[i]); } FieldDescriptionShort memberField{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; memberField.fieldName = "dims"sv; - FieldHeaderWriter::template put(buffer, memberField, dims); + FieldHeaderWriter::put(buffer, memberField, dims); memberField.fieldName = "values"sv; - FieldHeaderWriter::template put(buffer, memberField, value.elements()); + FieldHeaderWriter::put(buffer, memberField, value.elements()); buffer.resize(buffer.size() - 2); // remove trailing comma - buffer.put("\n}\n"sv); + buffer.put("\n}\n"sv); } static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { using namespace std::string_view_literals; DeserialiserInfo info{}; - FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = ExternalModifier::UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser>::getDataTypeId() || fieldHeader.fieldName != "dims"sv) { throw ProtocolException("expected multi-array dimensions"); } std::array dimWire; IoSerialiser>::deserialise(buffer, fieldHeader, dimWire); for (auto i = 0U; i < T::n_dims_; i++) { - value.dimensions()[i] = static_cast(dimWire[i]); + value.dimensions()[i] = static_cast(dimWire[i]); } value.element_count() = value.dimensions()[T::n_dims_ - 1]; value.stride(T::n_dims_ - 1) = 1; @@ -527,12 +528,12 @@ struct IoSerialiser { value.stride(i - 1) = value.stride(i) * value.dimensions()[i]; value.offset(i - 1) = 0; } - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser>::getDataTypeId() || fieldHeader.fieldName != "values"sv) { throw ProtocolException("expected multi-array values"); } IoSerialiser>::deserialise(buffer, fieldHeader, value.elements()); - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser::getDataTypeId()) { throw ProtocolException("expected end marker"); } @@ -542,8 +543,9 @@ struct IoSerialiser { template struct IoSerialiser> { // todo: arrays of objects - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } - inline constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::set &values) noexcept { + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::set &values) noexcept { buffer.put('['); bool first = true; for (auto &value : values) { @@ -569,7 +571,7 @@ struct IoSerialiser> { IoSerialiser::deserialise(buffer, field, entry); value.insert(entry); json::consumeWhitespace(buffer); - const auto next = buffer.template get(); + const auto next = buffer.get(); if (next == ']') { break; } @@ -583,30 +585,42 @@ struct IoSerialiser> { }; template requires(is_stringlike) struct IoSerialiser { - using K = typename T::key_type; - using V = typename T::mapped_type; - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER + using K = T::key_type; + using V = T::mapped_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &value) { using namespace std::string_view_literals; - buffer.put("{\n"sv); + buffer.put("{\n"sv); FieldDescriptionShort memberField{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; for (auto &entry : value) { memberField.fieldName = entry.first; - FieldHeaderWriter::template put(buffer, memberField, entry.second); + FieldHeaderWriter::put(buffer, memberField, entry.second); } buffer.resize(buffer.size() - 2); // remove trailing comma - buffer.put("\n}\n"sv); + buffer.put("\n}\n"sv); } static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { using namespace std::string_view_literals; DeserialiserInfo info{}; - FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = ExternalModifier::UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; - for (FieldHeaderReader::template get(buffer, info, fieldHeader); fieldHeader.intDataType != IoSerialiser::getDataTypeId(); FieldHeaderReader::get(buffer, info, fieldHeader)) { + FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; + for (FieldHeaderReader::get(buffer, info, fieldHeader); fieldHeader.intDataType != IoSerialiser::getDataTypeId(); FieldHeaderReader::get(buffer, info, fieldHeader)) { auto fieldValue = value[std::string(fieldHeader.fieldName)]; IoSerialiser::deserialise(buffer, fieldHeader, fieldValue); } } }; + +template +struct IoSerialiser { + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { + opencmw::serialise(buffer, value); + } + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, T &value) { + opencmw::deserialise(buffer, value); // TODO: refactor API to correctly propagate ProtocolCheck and Deserialiser Info + } +}; + } // namespace opencmw #endif // OPENCMW_JSONSERIALISER_H diff --git a/src/serialiser/test/IoSerialiserJson_tests.cpp b/src/serialiser/test/IoSerialiserJson_tests.cpp index 4bed5887..ae2784f9 100644 --- a/src/serialiser/test/IoSerialiserJson_tests.cpp +++ b/src/serialiser/test/IoSerialiserJson_tests.cpp @@ -128,11 +128,12 @@ TEST_CASE("JsonDeserialisationMissingField", "[JsonSerialiser]") { opencmw::debug::resetStats(); { opencmw::IoBuffer buffer; - buffer.put(R"({ "float1": 2.3, "superfluousField": { "p":12 , "a":null,"x" : false, "q": [ "a", "s"], "z": [true , false ] }, "test": { "intArray" : [ 1,2, 3], "val1":13.37e2, "val2":"bar"}, "int1": 42})"sv); + buffer.put(R"({ "float1": 2.3, "superfluousField": { "p":12 , "a":null,"x" : false, "q": [ "a", "s"], "z": [true , false ] }, "test": { "intArray" : [ 1,2, 3], "val1":13.37e2, "val2":"bar"}, "int1": 42} )"sv); std::cout << "Prepared json data: " << buffer.asString() << std::endl; Simple foo; auto result = opencmw::deserialise(buffer, foo); std::print(std::cout, "deserialisation finished: {}\n", result); + REQUIRE(buffer.position() == 189UZ); REQUIRE(foo.test.get()->val1 == 1337.0); REQUIRE(foo.test.get()->val2 == "bar"); REQUIRE(foo.test.get()->intArray == std::vector{ 1, 2, 3 }); From 382d5c2630801011bf5b1ca292ff074ef9315eb1 Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Mon, 9 Mar 2026 14:41:21 +0100 Subject: [PATCH 5/5] Zeromq message format: add utilities Topic: allow control over terminating pound sign. The pound sign is needed for zmq subscription topics to limit substring matching in case there are multiple subscriptions, but needs to be suppressed for looking up subscriptions in maps. ZmqUtils: allow to construct a message frame from a single char. Some zmq based protocols (e.g. rda3) make heavy use of zmq frames with a single byte, which otherwise use a lot of boilerplate code to construct these types of messages. Signed-off-by: Alexander Krimm --- src/core/include/Topic.hpp | 10 +++++++--- src/zmq/include/zmq/ZmqUtils.hpp | 10 ++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/core/include/Topic.hpp b/src/core/include/Topic.hpp index 6a851cb0..3fc228fa 100644 --- a/src/core/include/Topic.hpp +++ b/src/core/include/Topic.hpp @@ -103,11 +103,13 @@ struct Topic { return opencmw::URI::factory().path(_service).setQuery({ _params.begin(), _params.end() }).build(); } - std::string toZmqTopic() const { + std::string toZmqTopic(const bool delimitWithPound = true) const { using namespace std::string_literals; std::string zmqTopic = _service; if (_params.empty()) { - zmqTopic += "#"s; + if (delimitWithPound) { + zmqTopic += "#"s; + } return zmqTopic; } zmqTopic += "?"s; @@ -123,7 +125,9 @@ struct Topic { } isFirst = false; } - zmqTopic += "#"s; + if (delimitWithPound) { + zmqTopic += "#"s; + } return zmqTopic; } diff --git a/src/zmq/include/zmq/ZmqUtils.hpp b/src/zmq/include/zmq/ZmqUtils.hpp index 209bb369..d50571ba 100644 --- a/src/zmq/include/zmq/ZmqUtils.hpp +++ b/src/zmq/include/zmq/ZmqUtils.hpp @@ -231,6 +231,16 @@ class MessageFrame { copy); } + explicit MessageFrame(char c) { + const auto copy = new char[1] {c}; + zmq_msg_init_data( + &_message, copy, 1, + [](void * /*unused*/, void *bufOwned) { + delete static_cast(bufOwned); + }, + copy); + } + static MessageFrame fromStaticData(std::string_view buf) { MessageFrame mf; zmq_msg_init_data(