diff --git a/components/core/src/clp/ffi/ir_stream/Serializer.cpp b/components/core/src/clp/ffi/ir_stream/Serializer.cpp index 6a7128d9f..770a6a78c 100644 --- a/components/core/src/clp/ffi/ir_stream/Serializer.cpp +++ b/components/core/src/clp/ffi/ir_stream/Serializer.cpp @@ -1,22 +1,238 @@ #include "Serializer.hpp" +#include #include +#include +#include +#include +#include +#include #include +#include #include #include +#include #include "../../ir/types.hpp" #include "../../time_types.hpp" +#include "../../type_utils.hpp" #include "../encoding_methods.hpp" +#include "../SchemaTree.hpp" +#include "../SchemaTreeNode.hpp" #include "encoding_methods.hpp" #include "protocol_constants.hpp" #include "utils.hpp" +using std::optional; +using std::span; +using std::string; +using std::string_view; +using std::vector; + using clp::ir::eight_byte_encoded_variable_t; using clp::ir::four_byte_encoded_variable_t; namespace clp::ffi::ir_stream { +namespace { +/** + * Class for iterating the kv-pairs of a MessagePack map. + */ +class MsgpackMapIterator { +public: + // Types + using Child = msgpack::object_kv; + + // Constructors + MsgpackMapIterator(SchemaTreeNode::id_t schema_tree_node_id, span children) + : m_schema_tree_node_id{schema_tree_node_id}, + m_children{children}, + m_curr_child_it{m_children.begin()} {} + + // Methods + /** + * @return This map's ID in the schema tree. + */ + [[nodiscard]] auto get_schema_tree_node_id() const -> SchemaTreeNode::id_t { + return m_schema_tree_node_id; + } + + /** + * @return Whether there are more children to traverse. + */ + [[nodiscard]] auto has_next_child() const -> bool { + return m_curr_child_it != m_children.end(); + } + + /** + * Gets the next child and advances the underlying child idx. + * @return The next child to traverse. + */ + [[nodiscard]] auto get_next_child() -> Child const& { return *(m_curr_child_it++); } + +private: + SchemaTreeNode::id_t m_schema_tree_node_id; + span m_children; + span::iterator m_curr_child_it; +}; + +/** + * Gets the schema-tree node type that corresponds with a given MessagePack value. + * @param val + * @return The corresponding schema-tree node type. + * @return std::nullopt if the value doesn't match any of the supported schema-tree node types. + */ +[[nodiscard]] auto get_schema_tree_node_type_from_msgpack_val(msgpack::object const& val +) -> optional; + +/** + * Serializes an empty object. + * @param output_buf + */ +auto serialize_value_empty_object(vector& output_buf) -> void; + +/** + * Serializes an integer. + * @param val + * @param output_buf + * @return Whether serialization succeeded. + */ +auto serialize_value_int(int64_t val, vector& output_buf) -> void; + +/** + * Serializes a float. + * @param val + * @param output_buf + */ +auto serialize_value_float(double val, vector& output_buf) -> void; + +/** + * Serializes a boolean. + * @param val + * @param output_buf + */ +auto serialize_value_bool(bool val, vector& output_buf) -> void; + +/** + * Serializes a null. + * @param output_buf + */ +auto serialize_value_null(vector& output_buf) -> void; + +/** + * Serializes a string. + * @tparam encoded_variable_t + * @param val + * @param logtype_buf + * @param output_buf + * @return Whether serialization succeeded. + */ +template +[[nodiscard]] auto +serialize_value_string(string_view val, string& logtype_buf, vector& output_buf) -> bool; + +/** + * Serializes a MessagePack array as a JSON string, using CLP's encoding for unstructured text. + * @tparam encoded_variable_t + * @param val + * @param logtype_buf + * @param output_buf + * @return Whether serialization succeeded. + */ +template +[[nodiscard]] auto serialize_value_array( + msgpack::object const& val, + string& logtype_buf, + vector& output_buf +) -> bool; + +auto get_schema_tree_node_type_from_msgpack_val(msgpack::object const& val +) -> optional { + optional ret_val; + switch (val.type) { + case msgpack::type::POSITIVE_INTEGER: + case msgpack::type::NEGATIVE_INTEGER: + ret_val.emplace(SchemaTreeNode::Type::Int); + break; + case msgpack::type::FLOAT32: + case msgpack::type::FLOAT64: + ret_val.emplace(SchemaTreeNode::Type::Float); + break; + case msgpack::type::STR: + ret_val.emplace(SchemaTreeNode::Type::Str); + break; + case msgpack::type::BOOLEAN: + ret_val.emplace(SchemaTreeNode::Type::Bool); + break; + case msgpack::type::NIL: + case msgpack::type::MAP: + ret_val.emplace(SchemaTreeNode::Type::Obj); + break; + case msgpack::type::ARRAY: + ret_val.emplace(SchemaTreeNode::Type::UnstructuredArray); + break; + default: + return std::nullopt; + } + return ret_val; +} + +auto serialize_value_empty_object(vector& output_buf) -> void { + output_buf.push_back(cProtocol::Payload::ValueEmpty); +} + +auto serialize_value_int(int64_t val, vector& output_buf) -> void { + if (INT8_MIN <= val && val <= INT8_MAX) { + output_buf.push_back(cProtocol::Payload::ValueInt8); + output_buf.push_back(static_cast(val)); + } else if (INT16_MIN <= val && val <= INT16_MAX) { + output_buf.push_back(cProtocol::Payload::ValueInt16); + serialize_int(static_cast(val), output_buf); + } else if (INT32_MIN <= val && val <= INT32_MAX) { + output_buf.push_back(cProtocol::Payload::ValueInt32); + serialize_int(static_cast(val), output_buf); + } else { // (INT64_MIN <= val && val <= INT64_MAX) + output_buf.push_back(cProtocol::Payload::ValueInt64); + serialize_int(val, output_buf); + } +} + +auto serialize_value_float(double val, vector& output_buf) -> void { + output_buf.push_back(cProtocol::Payload::ValueFloat); + serialize_int(bit_cast(val), output_buf); +} + +auto serialize_value_bool(bool val, vector& output_buf) -> void { + output_buf.push_back(val ? cProtocol::Payload::ValueTrue : cProtocol::Payload::ValueFalse); +} + +auto serialize_value_null(vector& output_buf) -> void { + output_buf.push_back(cProtocol::Payload::ValueNull); +} + +template +auto serialize_value_string(string_view val, string& logtype_buf, vector& output_buf) + -> bool { + if (string_view::npos == val.find(' ')) { + return serialize_string(val, output_buf); + } + logtype_buf.clear(); + return serialize_clp_string(val, logtype_buf, output_buf); +} + +template +auto serialize_value_array( + msgpack::object const& val, + string& logtype_buf, + vector& output_buf +) -> bool { + std::ostringstream oss; + oss << val; + logtype_buf.clear(); + return serialize_clp_string(oss.str(), logtype_buf, output_buf); +} +} // namespace + template auto Serializer::create( ) -> BOOST_OUTCOME_V2_NAMESPACE::std_result> { @@ -59,14 +275,258 @@ auto Serializer::change_utc_offset(UtcOffset utc_offset) -> serialize_utc_offset_change(m_curr_utc_offset, m_ir_buf); } +template +auto Serializer::serialize_msgpack_map(msgpack::object_map const& msgpack_map +) -> bool { + if (0 == msgpack_map.size) { + serialize_value_empty_object(m_ir_buf); + return true; + } + + m_schema_tree.take_snapshot(); + m_schema_tree_node_buf.clear(); + m_key_group_buf.clear(); + m_value_group_buf.clear(); + + // Traverse the map using DFS iteratively + bool failure{false}; + vector dfs_stack; + dfs_stack.emplace_back( + SchemaTree::cRootId, + span{msgpack_map.ptr, msgpack_map.size} + ); + while (false == dfs_stack.empty()) { + auto& curr{dfs_stack.back()}; + if (false == curr.has_next_child()) { + // Visited all children, so pop node + dfs_stack.pop_back(); + continue; + } + + // Convert the current value's type to its corresponding schema-tree node type + auto const& [key, val]{curr.get_next_child()}; + auto const opt_schema_tree_node_type{get_schema_tree_node_type_from_msgpack_val(val)}; + if (false == opt_schema_tree_node_type.has_value()) { + failure = true; + break; + } + auto const schema_tree_node_type{opt_schema_tree_node_type.value()}; + + SchemaTree::NodeLocator const locator{ + curr.get_schema_tree_node_id(), + key.as(), + schema_tree_node_type + }; + + // Get the schema-tree node that corresponds with the current kv-pair, or add it if it + // doesn't exist. + auto opt_schema_tree_node_id{m_schema_tree.try_get_node_id(locator)}; + if (false == opt_schema_tree_node_id.has_value()) { + opt_schema_tree_node_id.emplace(m_schema_tree.insert_node(locator)); + if (false == serialize_schema_tree_node(locator)) { + failure = true; + break; + } + } + auto const schema_tree_node_id{opt_schema_tree_node_id.value()}; + + if (msgpack::type::MAP == val.type) { + // Serialize map + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) + auto const& inner_map{val.via.map}; + auto const inner_map_size(inner_map.size); + if (0 == inner_map_size) { + // Value is an empty map, so we can serialize it immediately + if (false == serialize_key(schema_tree_node_id)) { + failure = true; + break; + } + serialize_value_empty_object(m_value_group_buf); + } else { + // Add map for DFS iteration + dfs_stack.emplace_back( + schema_tree_node_id, + span{inner_map.ptr, inner_map_size} + ); + } + } else { + // Serialize primitive + if (false + == (serialize_key(schema_tree_node_id) && serialize_val(val, schema_tree_node_type) + )) + { + failure = true; + break; + } + } + } + + if (failure) { + m_schema_tree.revert(); + return false; + } + + m_ir_buf.insert( + m_ir_buf.cend(), + m_schema_tree_node_buf.cbegin(), + m_schema_tree_node_buf.cend() + ); + m_ir_buf.insert(m_ir_buf.cend(), m_key_group_buf.cbegin(), m_key_group_buf.cend()); + m_ir_buf.insert(m_ir_buf.cend(), m_value_group_buf.cbegin(), m_value_group_buf.cend()); + return true; +} + +template +auto Serializer::serialize_schema_tree_node( + SchemaTree::NodeLocator const& locator +) -> bool { + switch (locator.get_type()) { + case SchemaTreeNode::Type::Int: + m_schema_tree_node_buf.push_back(cProtocol::Payload::SchemaTreeNodeInt); + break; + case SchemaTreeNode::Type::Float: + m_schema_tree_node_buf.push_back(cProtocol::Payload::SchemaTreeNodeFloat); + break; + case SchemaTreeNode::Type::Bool: + m_schema_tree_node_buf.push_back(cProtocol::Payload::SchemaTreeNodeBool); + break; + case SchemaTreeNode::Type::Str: + m_schema_tree_node_buf.push_back(cProtocol::Payload::SchemaTreeNodeStr); + break; + case SchemaTreeNode::Type::UnstructuredArray: + m_schema_tree_node_buf.push_back(cProtocol::Payload::SchemaTreeNodeUnstructuredArray); + break; + case SchemaTreeNode::Type::Obj: + m_schema_tree_node_buf.push_back(cProtocol::Payload::SchemaTreeNodeObj); + break; + default: + // Unknown type + return false; + } + + auto const parent_id{locator.get_parent_id()}; + if (parent_id <= UINT8_MAX) { + m_schema_tree_node_buf.push_back(cProtocol::Payload::SchemaTreeNodeParentIdUByte); + m_schema_tree_node_buf.push_back(bit_cast(static_cast(parent_id))); + } else if (parent_id <= UINT16_MAX) { + m_schema_tree_node_buf.push_back(cProtocol::Payload::SchemaTreeNodeParentIdUShort); + serialize_int(static_cast(parent_id), m_schema_tree_node_buf); + } else { + // Out of range + return false; + } + + return serialize_string(locator.get_key_name(), m_schema_tree_node_buf); +} + +template +auto Serializer::serialize_key(SchemaTreeNode::id_t id) -> bool { + if (id <= UINT8_MAX) { + m_key_group_buf.push_back(cProtocol::Payload::KeyIdUByte); + m_key_group_buf.push_back(bit_cast(static_cast(id))); + } else if (id <= UINT16_MAX) { + m_key_group_buf.push_back(cProtocol::Payload::KeyIdUShort); + serialize_int(static_cast(id), m_key_group_buf); + } else { + return false; + } + return true; +} + +template +auto Serializer::serialize_val( + msgpack::object const& val, + SchemaTreeNode::Type schema_tree_node_type +) -> bool { + switch (schema_tree_node_type) { + case SchemaTreeNode::Type::Int: + if (msgpack::type::POSITIVE_INTEGER == val.type + && static_cast(INT64_MAX) < val.as()) + { + return false; + } + serialize_value_int(val.as(), m_value_group_buf); + break; + + case SchemaTreeNode::Type::Float: + serialize_value_float(val.as(), m_value_group_buf); + break; + + case SchemaTreeNode::Type::Bool: + serialize_value_bool(val.as(), m_value_group_buf); + break; + + case SchemaTreeNode::Type::Str: + if (false + == serialize_value_string( + val.as(), + m_logtype_buf, + m_value_group_buf + )) + { + return false; + } + break; + + case SchemaTreeNode::Type::Obj: + if (msgpack::type::NIL != val.type) { + return false; + } + serialize_value_null(m_value_group_buf); + break; + + case SchemaTreeNode::Type::UnstructuredArray: + if (false + == serialize_value_array(val, m_logtype_buf, m_value_group_buf)) + { + return false; + } + break; + + default: + // Unknown schema tree node type + return false; + } + return true; +} + // Explicitly declare template specializations so that we can define the template methods in this // file template auto Serializer::create( ) -> BOOST_OUTCOME_V2_NAMESPACE::std_result>; template auto Serializer::create( ) -> BOOST_OUTCOME_V2_NAMESPACE::std_result>; + template auto Serializer::change_utc_offset(UtcOffset utc_offset ) -> void; template auto Serializer::change_utc_offset(UtcOffset utc_offset ) -> void; + +template auto Serializer::serialize_msgpack_map( + msgpack::object_map const& msgpack_map +) -> bool; +template auto Serializer::serialize_msgpack_map( + msgpack::object_map const& msgpack_map +) -> bool; + +template auto Serializer::serialize_schema_tree_node( + SchemaTree::NodeLocator const& locator +) -> bool; +template auto Serializer::serialize_schema_tree_node( + SchemaTree::NodeLocator const& locator +) -> bool; + +template auto Serializer::serialize_key(SchemaTreeNode::id_t id +) -> bool; +template auto Serializer::serialize_key(SchemaTreeNode::id_t id +) -> bool; + +template auto Serializer::serialize_val( + msgpack::object const& val, + SchemaTreeNode::Type schema_tree_node_type +) -> bool; +template auto Serializer::serialize_val( + msgpack::object const& val, + SchemaTreeNode::Type schema_tree_node_type +) -> bool; } // namespace clp::ffi::ir_stream diff --git a/components/core/src/clp/ffi/ir_stream/Serializer.hpp b/components/core/src/clp/ffi/ir_stream/Serializer.hpp index 4c8d48fc6..82863bc7e 100644 --- a/components/core/src/clp/ffi/ir_stream/Serializer.hpp +++ b/components/core/src/clp/ffi/ir_stream/Serializer.hpp @@ -3,12 +3,15 @@ #include #include +#include #include #include +#include #include "../../time_types.hpp" #include "../SchemaTree.hpp" +#include "../SchemaTreeNode.hpp" namespace clp::ffi::ir_stream { /** @@ -79,14 +82,47 @@ class Serializer { */ auto change_utc_offset(UtcOffset utc_offset) -> void; + /** + * Serializes the given msgpack map as a key-value pair log event. + * @param msgpack_map + * @return Whether serialization succeeded. + */ + [[nodiscard]] auto serialize_msgpack_map(msgpack::object_map const& msgpack_map) -> bool; + private: // Constructors Serializer() = default; + // Methods + /** + * Serializes a schema tree node identified by the given locator into `m_schema_tree_node_buf`. + * @param locator + * @return Whether serialization succeeded. + */ + [[nodiscard]] auto serialize_schema_tree_node(SchemaTree::NodeLocator const& locator) -> bool; + + /** + * Serializes the given key ID into `m_key_group_buf`. + * @param id + * @return true on success. + * @return false if the ID exceeds the representable range. + */ + [[nodiscard]] auto serialize_key(SchemaTreeNode::id_t id) -> bool; + + /** + * Serializes the given MessagePack value into `m_value_group_buf`. + * @param val + * @param schema_tree_node_type The type of the schema tree node that corresponds to `val`. + * @return Whether serialization succeeded. + */ + [[nodiscard]] auto + serialize_val(msgpack::object const& val, SchemaTreeNode::Type schema_tree_node_type) -> bool; + UtcOffset m_curr_utc_offset{0}; Buffer m_ir_buf; SchemaTree m_schema_tree; + std::string m_logtype_buf; Buffer m_schema_tree_node_buf; Buffer m_key_group_buf; Buffer m_value_group_buf; diff --git a/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp b/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp index 35363e2d3..128d659c1 100644 --- a/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp +++ b/components/core/src/clp/ffi/ir_stream/encoding_methods.cpp @@ -135,6 +135,22 @@ bool serialize_log_event( string_view message, string& logtype, vector& ir_buf +) { + if (false == serialize_message(message, logtype, ir_buf)) { + return false; + } + + // Encode timestamp + ir_buf.push_back(cProtocol::Payload::TimestampVal); + serialize_int(timestamp, ir_buf); + + return true; +} + +bool serialize_message( + std::string_view message, + std::string& logtype, + std::vector& ir_buf ) { auto encoded_var_handler = [&ir_buf](eight_byte_encoded_variable_t encoded_var) { ir_buf.push_back(cProtocol::Payload::VarEightByteEncoding); @@ -153,15 +169,7 @@ bool serialize_log_event( return false; } - if (false == serialize_logtype(logtype, ir_buf)) { - return false; - } - - // Encode timestamp - ir_buf.push_back(cProtocol::Payload::TimestampVal); - serialize_int(timestamp, ir_buf); - - return true; + return serialize_logtype(logtype, ir_buf); } } // namespace eight_byte_encoding diff --git a/components/core/src/clp/ffi/ir_stream/encoding_methods.hpp b/components/core/src/clp/ffi/ir_stream/encoding_methods.hpp index 0129e7550..be042e870 100644 --- a/components/core/src/clp/ffi/ir_stream/encoding_methods.hpp +++ b/components/core/src/clp/ffi/ir_stream/encoding_methods.hpp @@ -39,6 +39,15 @@ bool serialize_log_event( std::string& logtype, std::vector& ir_buf ); + +/** + * Serializes the given message into the eight-byte encoding IR stream. + * @param message + * @param logtype Returns the message's logtype. + * @param ir_buf + * @return Whether the message was serialized successfully. + */ +bool serialize_message(std::string_view message, std::string& logtype, std::vector& ir_buf); } // namespace eight_byte_encoding namespace four_byte_encoding { diff --git a/components/core/src/clp/ffi/ir_stream/protocol_constants.hpp b/components/core/src/clp/ffi/ir_stream/protocol_constants.hpp index f9fd7ed78..9f84cc4e1 100644 --- a/components/core/src/clp/ffi/ir_stream/protocol_constants.hpp +++ b/components/core/src/clp/ffi/ir_stream/protocol_constants.hpp @@ -50,6 +50,35 @@ constexpr int8_t TimestampDeltaInt = 0x33; constexpr int8_t TimestampDeltaLong = 0x34; constexpr int8_t UtcOffsetChange = 0x3F; + +constexpr int8_t StrLenUByte = 0x41; +constexpr int8_t StrLenUShort = 0x42; +constexpr int8_t StrLenUInt = 0x43; + +constexpr int8_t ValueInt8 = 0x51; +constexpr int8_t ValueInt16 = 0x52; +constexpr int8_t ValueInt32 = 0x53; +constexpr int8_t ValueInt64 = 0x54; +constexpr int8_t ValueFloat = 0x56; +constexpr int8_t ValueTrue = 0x57; +constexpr int8_t ValueFalse = 0x58; +constexpr int8_t ValueFourByteEncodingClpStr = 0x59; +constexpr int8_t ValueEightByteEncodingClpStr = 0x5A; +constexpr int8_t ValueEmpty = 0x5E; +constexpr int8_t ValueNull = 0x5F; + +constexpr int8_t SchemaTreeNodeParentIdUByte = 0x60; +constexpr int8_t SchemaTreeNodeParentIdUShort = 0x61; + +constexpr int8_t KeyIdUByte = 0x65; +constexpr int8_t KeyIdUShort = 0x66; + +constexpr int8_t SchemaTreeNodeInt = 0x71; +constexpr int8_t SchemaTreeNodeFloat = 0x72; +constexpr int8_t SchemaTreeNodeBool = 0x73; +constexpr int8_t SchemaTreeNodeStr = 0x74; +constexpr int8_t SchemaTreeNodeUnstructuredArray = 0x75; +constexpr int8_t SchemaTreeNodeObj = 0x76; } // namespace Payload constexpr int8_t FourByteEncodingMagicNumber[] diff --git a/components/core/src/clp/ffi/ir_stream/utils.cpp b/components/core/src/clp/ffi/ir_stream/utils.cpp index 0a22536b6..66277720c 100644 --- a/components/core/src/clp/ffi/ir_stream/utils.cpp +++ b/components/core/src/clp/ffi/ir_stream/utils.cpp @@ -1,6 +1,7 @@ #include "utils.hpp" #include +#include #include #include @@ -9,24 +10,43 @@ #include "protocol_constants.hpp" namespace clp::ffi::ir_stream { -auto serialize_metadata(nlohmann::json& metadata, std::vector& ir_buf) -> bool { - ir_buf.push_back(cProtocol::Metadata::EncodingJson); +auto serialize_metadata(nlohmann::json& metadata, std::vector& output_buf) -> bool { + output_buf.push_back(cProtocol::Metadata::EncodingJson); auto const metadata_serialized = metadata.dump(-1, ' ', false, nlohmann::json::error_handler_t::ignore); auto const metadata_serialized_length = metadata_serialized.length(); if (metadata_serialized_length <= UINT8_MAX) { - ir_buf.push_back(cProtocol::Metadata::LengthUByte); - ir_buf.push_back(bit_cast(static_cast(metadata_serialized_length))); + output_buf.push_back(cProtocol::Metadata::LengthUByte); + output_buf.push_back(bit_cast(static_cast(metadata_serialized_length))); } else if (metadata_serialized_length <= UINT16_MAX) { - ir_buf.push_back(cProtocol::Metadata::LengthUShort); - serialize_int(static_cast(metadata_serialized_length), ir_buf); + output_buf.push_back(cProtocol::Metadata::LengthUShort); + serialize_int(static_cast(metadata_serialized_length), output_buf); } else { // Can't encode metadata longer than 64 KiB return false; } - ir_buf.insert(ir_buf.cend(), metadata_serialized.cbegin(), metadata_serialized.cend()); + output_buf.insert(output_buf.cend(), metadata_serialized.cbegin(), metadata_serialized.cend()); return true; } + +auto serialize_string(std::string_view str, std::vector& output_buf) -> bool { + auto const length{str.length()}; + if (length <= UINT8_MAX) { + output_buf.push_back(cProtocol::Payload::StrLenUByte); + output_buf.push_back(bit_cast(static_cast(length))); + } else if (length <= UINT16_MAX) { + output_buf.push_back(cProtocol::Payload::StrLenUShort); + serialize_int(static_cast(length), output_buf); + } else if (length <= UINT32_MAX) { + output_buf.push_back(cProtocol::Payload::StrLenUInt); + serialize_int(static_cast(length), output_buf); + } else { + // Out of range + return false; + } + output_buf.insert(output_buf.cend(), str.cbegin(), str.cend()); + return true; +} } // namespace clp::ffi::ir_stream diff --git a/components/core/src/clp/ffi/ir_stream/utils.hpp b/components/core/src/clp/ffi/ir_stream/utils.hpp index 5e1fb293a..7879c3f74 100644 --- a/components/core/src/clp/ffi/ir_stream/utils.hpp +++ b/components/core/src/clp/ffi/ir_stream/utils.hpp @@ -3,33 +3,61 @@ #include #include +#include +#include #include #include +#include "../../ir/types.hpp" #include "byteswap.hpp" +#include "encoding_methods.hpp" +#include "protocol_constants.hpp" namespace clp::ffi::ir_stream { /** * Serializes the given metadata into the IR stream. * @param metadata - * @param ir_buf + * @param output_buf * @return Whether serialization succeeded. */ [[nodiscard]] auto -serialize_metadata(nlohmann::json& metadata, std::vector& ir_buf) -> bool; +serialize_metadata(nlohmann::json& metadata, std::vector& output_buf) -> bool; /** * Serializes the given integer into the IR stream. * @tparam integer_t * @param value - * @param ir_buf + * @param output_buf */ template -auto serialize_int(integer_t value, std::vector& ir_buf) -> void; +auto serialize_int(integer_t value, std::vector& output_buf) -> void; + +/** + * Serializes a string using CLP's encoding for unstructured text. + * @tparam encoded_variable_t + * @param str + * @param logtype Returns the corresponding logtype. + * @param output_buf + * @return Whether serialization succeeded. + */ +template +[[nodiscard]] auto serialize_clp_string( + std::string_view str, + std::string& logtype, + std::vector& output_buf +) -> bool; + +/** + * Serializes a string. + * @param str + * @param output_buf + * @return Whether serialization succeeded. + */ +[[nodiscard]] auto serialize_string(std::string_view str, std::vector& output_buf) -> bool; template -auto serialize_int(integer_t value, std::vector& ir_buf) -> void { +auto serialize_int(integer_t value, std::vector& output_buf) -> void { integer_t value_big_endian{}; static_assert(sizeof(integer_t) == 2 || sizeof(integer_t) == 4 || sizeof(integer_t) == 8); if constexpr (sizeof(value) == 2) { @@ -41,7 +69,28 @@ auto serialize_int(integer_t value, std::vector& ir_buf) -> void { } // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) std::span const data_view{reinterpret_cast(&value_big_endian), sizeof(value)}; - ir_buf.insert(ir_buf.end(), data_view.begin(), data_view.end()); + output_buf.insert(output_buf.end(), data_view.begin(), data_view.end()); +} + +template +[[nodiscard]] auto serialize_clp_string( + std::string_view str, + std::string& logtype, + std::vector& output_buf +) -> bool { + static_assert( + (std::is_same_v + || std::is_same_v) + ); + bool succeeded{}; + if constexpr (std::is_same_v) { + output_buf.push_back(cProtocol::Payload::ValueFourByteEncodingClpStr); + succeeded = four_byte_encoding::serialize_message(str, logtype, output_buf); + } else { + output_buf.push_back(cProtocol::Payload::ValueEightByteEncodingClpStr); + succeeded = eight_byte_encoding::serialize_message(str, logtype, output_buf); + } + return succeeded; } } // namespace clp::ffi::ir_stream #endif diff --git a/components/core/tests/test-ir_encoding_methods.cpp b/components/core/tests/test-ir_encoding_methods.cpp index 167803e58..4199428ff 100644 --- a/components/core/tests/test-ir_encoding_methods.cpp +++ b/components/core/tests/test-ir_encoding_methods.cpp @@ -1,9 +1,14 @@ +#include +#include #include +#include +#include #include #include #include #include +#include #include "../src/clp/BufferReader.hpp" #include "../src/clp/ErrorCode.hpp" @@ -132,6 +137,19 @@ auto flush_and_clear_serializer_buffer( std::vector& byte_buf ) -> void; +/** + * Unpacks and serializes the given msgpack bytes using kv serializer. + * @tparam encoded_variable_t + * @param msgpack_bytes + * @param serializer + * @return Whether serialization succeeded. + */ +template +[[nodiscard]] auto unpack_and_serialize_msgpack_bytes( + vector const& msgpack_bytes, + Serializer& serializer +) -> bool; + template [[nodiscard]] auto serialize_log_events( vector const& log_events, @@ -241,6 +259,22 @@ auto flush_and_clear_serializer_buffer( byte_buf.insert(byte_buf.cend(), view.begin(), view.end()); serializer.clear_ir_buf(); } + +template +auto unpack_and_serialize_msgpack_bytes( + vector const& msgpack_bytes, + Serializer& serializer +) -> bool { + auto const msgpack_obj_handle{msgpack::unpack( + clp::size_checked_pointer_cast(msgpack_bytes.data()), + msgpack_bytes.size() + )}; + auto const msgpack_obj{msgpack_obj_handle.get()}; + if (msgpack::type::MAP != msgpack_obj.type) { + return false; + } + return serializer.serialize_msgpack_map(msgpack_obj.via.map); +} } // namespace /** @@ -971,3 +1005,83 @@ TEMPLATE_TEST_CASE( && 0 == num_bytes_read) ); } + +TEMPLATE_TEST_CASE( + "ffi_ir_stream_Serializer_serialize_msgpack", + "[clp][ffi][ir_stream][Serializer]", + four_byte_encoded_variable_t, + eight_byte_encoded_variable_t +) { + // TODO: Test deserializing the serialized bytes once a KV-pair IR deserializer is implemented. + + vector ir_buf; + + auto result{Serializer::create()}; + REQUIRE((false == result.has_error())); + + auto& serializer{result.value()}; + flush_and_clear_serializer_buffer(serializer, ir_buf); + + auto const empty_obj = nlohmann::json::parse("{}"); + REQUIRE(unpack_and_serialize_msgpack_bytes(nlohmann::json::to_msgpack(empty_obj), serializer)); + + // Test encoding basic object + constexpr string_view cShortString{"short_string"}; + constexpr string_view cClpString{"uid=0, CPU usage: 99.99%, \"user_name\"=YScope"}; + auto const empty_array = nlohmann::json::parse("[]"); + nlohmann::json const basic_obj + = {{"int8_max", INT8_MAX}, + {"int8_min", INT8_MIN}, + {"int16_max", INT16_MAX}, + {"int16_min", INT16_MIN}, + {"int32_max", INT32_MAX}, + {"int32_min", INT32_MIN}, + {"int64_max", INT64_MAX}, + {"int64_min", INT64_MIN}, + {"float_zero", 0.0}, + {"float_pos", 1.01}, + {"float_neg", -1.01}, + {"true", true}, + {"false", false}, + {"string", cShortString}, + {"clp_string", cClpString}, + {"null", nullptr}, + {"empty_object", empty_obj}, + {"empty_array", empty_array}}; + + REQUIRE(unpack_and_serialize_msgpack_bytes(nlohmann::json::to_msgpack(basic_obj), serializer)); + + auto basic_array = empty_array; + basic_array.emplace_back(1); + basic_array.emplace_back(1.0); + basic_array.emplace_back(true); + basic_array.emplace_back(cShortString); + basic_array.emplace_back(cClpString); + basic_array.emplace_back(nullptr); + basic_array.emplace_back(empty_array); + for (auto const& element : basic_array) { + // Non-map objects should not be serializable + REQUIRE( + (false + == unpack_and_serialize_msgpack_bytes( + nlohmann::json::to_msgpack(element), + serializer + )) + ); + } + basic_array.emplace_back(empty_obj); + + // Recursively construct an object containing inner maps and inner arrays. + auto recursive_obj = basic_obj; + auto recursive_array = basic_array; + constexpr size_t cRecursiveDepth{6}; + for (size_t i{0}; i < cRecursiveDepth; ++i) { + recursive_array.emplace_back(recursive_obj); + recursive_obj.emplace("obj_" + std::to_string(i), recursive_obj); + recursive_obj.emplace("array_" + std::to_string(i), recursive_array); + REQUIRE(unpack_and_serialize_msgpack_bytes( + nlohmann::json::to_msgpack(recursive_obj), + serializer + )); + } +}