From 286eac478b3242f82d79ca45c9730d9f3127225f Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 16:11:37 +0800 Subject: [PATCH 01/32] add support for big-endian --- include/ylt/struct_pack/endian_wrapper.hpp | 218 ++++++++++++++++++ include/ylt/struct_pack/marco.h | 8 + include/ylt/struct_pack/packer.hpp | 105 ++++++--- include/ylt/struct_pack/reflection.hpp | 3 + include/ylt/struct_pack/unpacker.hpp | 147 ++++++++---- include/ylt/struct_pack/varint.hpp | 4 +- .../tests/binary_data/test_cross_platform.dat | Bin 255 -> 362 bytes ...test_cross_platform_without_debug_info.dat | Bin 184 -> 281 bytes src/struct_pack/tests/test_cross_platform.cpp | 13 ++ src/struct_pack/tests/test_struct.hpp | 45 ++-- 10 files changed, 451 insertions(+), 92 deletions(-) create mode 100644 include/ylt/struct_pack/endian_wrapper.hpp diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp new file mode 100644 index 000000000..6797ab262 --- /dev/null +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023, Alibaba Group Holding Limited; + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include "reflection.hpp" +#include "ylt/struct_pack/util.h" +namespace struct_pack::detail { + +template +void write_wrapper(writer_t& writer, const char* data) { + if constexpr (is_system_little_endian || block_size == 1) { + writer.write(data, block_size); + } + else if constexpr (block_size == 2) { +#ifdef _MSC_VER + auto tmp = _byteswap_ushort(*data); + writer.write(&tmp, block_size); +#elif defined(__clang__) || defined(__GNUC__) + auto tmp = __builtin_bswap16(*data); + writer.write(&tmp, block_size); +#else + writer.write(data + 1, 1); + writer.write(data, 1); +#endif + } + else if constexpr (block_size == 4) { +#ifdef _MSC_VER + auto tmp = _byteswap_ulong(*data); + writer.write(&tmp, block_size); +#elif defined(__clang__) || defined(__GNUC__) + auto tmp = __builtin_bswap16(*data); + writer.write(&tmp, block_size); +#else + writer.write(data + 3, 1); + writer.write(data + 2, 1); + writer.write(data + 1, 1); + writer.write(data, 1); +#endif + } + else if constexpr (block_size == 8) { +#ifdef _MSC_VER + auto tmp = _byteswap_uint64(*data); + writer.write(&tmp, block_size); +#elif defined(__clang__) || defined(__GNUC__) + auto tmp = __builtin_bswap16(*data); + writer.write(&tmp, block_size); +#else + writer.write(data + 7, 1); + writer.write(data + 6, 1); + writer.write(data + 5, 1); + writer.write(data + 4, 1); + writer.write(data + 3, 1); + writer.write(data + 2, 1); + writer.write(data + 1, 1); + writer.write(data, 1); +#endif + } + else if constexpr (block_size == 16) { +#ifdef _MSC_VER + auto tmp1 = _byteswap_uint64(*data), tmp2 = _byteswap_uint64(*data + 8); + writer.write(&tmp2, block_size); + writer.write(&tmp1, block_size); +#elif defined(__clang__) || defined(__GNUC__) + auto tmp1 = __builtin_bswap64(*data), tmp2 = __builtin_bswap64(*data + 8); + writer.write(&tmp2, block_size); + writer.write(&tmp1, block_size); +#else + writer.write(data + 15, 1); + writer.write(data + 14, 1); + writer.write(data + 13, 1); + writer.write(data + 12, 1); + writer.write(data + 11, 1); + writer.write(data + 10, 1); + writer.write(data + 9, 1); + writer.write(data + 8, 1); + writer.write(data + 7, 1); + writer.write(data + 6, 1); + writer.write(data + 5, 1); + writer.write(data + 4, 1); + writer.write(data + 3, 1); + writer.write(data + 2, 1); + writer.write(data + 1, 1); + writer.write(data, 1); +#endif + } + else { + static_assert(sizeof(writer), "illegal block size(should be 1,2,4,8,16)"); + } +} +template +void write_wrapper(writer_t& writer, const char* data, + std::size_t block_count) { + if constexpr (is_system_little_endian || block_size == 1) { + auto total_sz = block_count * block_size; + if SP_UNLIKELY (total_sz >= PTRDIFF_MAX) + unreachable(); + else + writer.write(data, total_sz); + } + else { + for (std::size_t i = 0; i < block_count; ++i) { + write_wrapper(writer, data); + data += block_size; + } + } +} +template +bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { + if constexpr (is_system_little_endian || block_size == 1) { + return static_cast(reader.read(data, block_size)); + } + else { + std::array tmp; + bool res = static_cast(reader.read(&tmp, block_size)); + if SP_UNLIKELY (res) { + return res; + } + if constexpr (block_size == 2) { +#ifdef _MSC_VER + *(uint16_t*)data = _byteswap_ushort(*(uint16_t*)&tmp); +#elif defined(__clang__) || defined(__GNUC__) + *(uint16_t*)data = __builtin_bswap16(*(uint16_t*)&tmp); +#else + data[0] = tmp[1]; + data[1] = tmp[0]; +#endif + } + else if constexpr (block_size == 4) { +#ifdef _MSC_VER + *(uint32_t*)data = _byteswap_ulong(*(uint32_t*)&tmp); +#elif defined(__clang__) || defined(__GNUC__) + *(uint32_t*)data = __builtin_bswap32(*(uint32_t*)&tmp); +#else + data[0] = tmp[3]; + data[1] = tmp[2]; + data[2] = tmp[1]; + data[3] = tmp[0]; +#endif + } + else if constexpr (block_size == 8) { +#ifdef _MSC_VER + *(uint64_t*)data = _byteswap_uint64(*(uint64_t*)&tmp); +#elif defined(__clang__) || defined(__GNUC__) + *(uint64_t*)data = __builtin_bswap64(*(uint64_t*)&tmp); +#else + data[0] = tmp[7]; + data[1] = tmp[6]; + data[2] = tmp[5]; + data[3] = tmp[4]; + data[4] = tmp[3]; + data[5] = tmp[2]; + data[6] = tmp[1]; + data[7] = tmp[0]; +#endif + } + else if constexpr (block_size == 16) { +#ifdef _MSC_VER + *(uint64_t*)(data + 8) = _byteswap_uint64(*(uint64_t*)&tmp); + *(uint64_t*)data = _byteswap_uint64(*(uint64_t*)(&tmp + 8)); +#elif defined(__clang__) || defined(__GNUC__) + *(uint64_t*)(data + 8) = __builtin_bswap64(*(uint64_t*)&tmp); + *(uint64_t*)data = __builtin_bswap64(*(uint64_t*)(&tmp + 8)); +#else + data[0] = tmp[15]; + data[1] = tmp[14]; + data[2] = tmp[13]; + data[3] = tmp[12]; + data[4] = tmp[11]; + data[5] = tmp[10]; + data[6] = tmp[9]; + data[7] = tmp[8]; + data[8] = tmp[7]; + data[9] = tmp[6]; + data[10] = tmp[5]; + data[11] = tmp[4]; + data[12] = tmp[3]; + data[13] = tmp[2]; + data[14] = tmp[1]; + data[15] = tmp[0]; +#endif + } + else { + static_assert(sizeof(reader), "illegal block size(should be 1,2,4,8,16)"); + } + } +} +template +bool read_wrapper(reader_t& reader, char* SP_RESTRICT data, + std::size_t block_count) { + if constexpr (is_system_little_endian || block_size == 1) { + auto total_sz = block_count * block_size; + if SP_UNLIKELY (total_sz >= PTRDIFF_MAX) + unreachable(); + else + return static_cast(reader.read(data, total_sz)); + } + else { + for (std::size_t i = 0; i < block_count; ++i) { + if SP_UNLIKELY (!read_wrapper(reader, data)) { + return false; + } + data += block_size; + } + } +} +}; // namespace struct_pack::detail \ No newline at end of file diff --git a/include/ylt/struct_pack/marco.h b/include/ylt/struct_pack/marco.h index 7dbc0a243..546ac6e9e 100644 --- a/include/ylt/struct_pack/marco.h +++ b/include/ylt/struct_pack/marco.h @@ -60,3 +60,11 @@ #define STRUCT_PACK_RTTI_ENABLED #endif #endif + +#if defined __clang__ || __GNUC__ +#define SP_RESTRICT __restrict__ +#elif defined _MSC_VER +#define SP_RESTRICT __restrict +#else +#define SP_RESTRICT +#endif diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index c6103e392..eafce4ba3 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -14,8 +14,12 @@ * limitations under the License. */ #pragma once +#include + #include "calculate_size.hpp" #include "reflection.hpp" +#include "endian_wrapper.hpp" +#include "ylt/struct_pack/util.h" namespace struct_pack::detail { template < #if __cpp_concepts >= 201907L @@ -88,20 +92,35 @@ class packer { constexpr void STRUCT_PACK_INLINE serialize_metainfo() { constexpr auto hash_head = calculate_hash_head() | (is_default_size_type ? 0 : 1); - writer_.write((char *)&hash_head, sizeof(uint32_t)); + write_wrapper(writer_, (char *)&hash_head); if constexpr (hash_head % 2) { // has more metainfo auto metainfo = info.metainfo(); - writer_.write((char *)&metainfo, sizeof(char)); + write_wrapper(writer_, (char *)&metainfo); if constexpr (serialize_static_config::has_compatible) { - constexpr std::size_t sz[] = {0, 2, 4, 8}; - auto len_size = sz[metainfo & 0b11]; - auto len = info.size(); - writer_.write((char *)&len, len_size); + uint16_t len16; + uint32_t len32; + uint64_t len64; + switch (metainfo & 0b11) { + case 1: + len16 = info.size(); + write_wrapper<2>(writer_, (char *)&len16); + break; + case 2: + len32 = info.size(); + write_wrapper<4>(writer_, (char *)&len32); + break; + case 3: + len64 = info.size(); + write_wrapper<8>(writer_, (char *)&len64); + break; + default: + unreachable(); + } } if constexpr (check_if_add_type_literal()) { constexpr auto type_literal = struct_pack::get_type_literal(); - writer_.write(type_literal.data(), type_literal.size() + 1); + write_wrapper<1>(writer_, type_literal.data(), type_literal.size() + 1); } } } @@ -119,7 +138,7 @@ class packer { constexpr void STRUCT_PACK_INLINE write_padding(std::size_t sz) { if (sz > 0) { constexpr char buf = 0; - for (std::size_t i = 0; i < sz; ++i) writer_.write(&buf, 1); + for (std::size_t i = 0; i < sz; ++i) write_wrapper<1>(writer_, &buf); } } @@ -141,11 +160,11 @@ class packer { else if constexpr (std::is_fundamental_v || std::is_enum_v || id == type_id::int128_t || id == type_id::uint128_t || id == type_id::bitset_t) { - writer_.write((char *)&item, sizeof(type)); + write_wrapper(writer_, (char *)&item); } else if constexpr (unique_ptr) { bool has_value = (item != nullptr); - writer_.write((char *)&has_value, sizeof(char)); + write_wrapper(writer_, (char *)&has_value); if (has_value) { if constexpr (is_base_class) { bool is_ok; @@ -153,7 +172,7 @@ class packer { auto index = search_type_by_md5( item->get_struct_pack_id(), is_ok); assert(is_ok); - writer_.write((char *)&id, sizeof(uint32_t)); + write_wrapper(writer_, (char *)&id); template_switch, std::integral_constant, @@ -170,7 +189,8 @@ class packer { } else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value) { - writer_.write((char *)&item, sizeof(type)); + write_wrapper( + writer_, (char *)&item, sizeof(type) / sizeof(decltype(item[0]))); } else { for (const auto &i : item) { @@ -179,23 +199,43 @@ class packer { } } else if constexpr (map_container || container) { - uint64_t size = item.size(); -#ifdef STRUCT_PACK_OPTIMIZE - writer_.write((char *)&size, size_type); -#else if constexpr (size_type == 1) { - writer_.write((char *)&size, size_type); + uint8_t size = item.size(); + write_wrapper(writer_, (char *)&size); + } +#ifdef STRUCT_PACK_OPTIMIZE + else if constexpr (size_type == 2) { + uint16_t size = item.size(); + write_wrapper(writer_, (char *)&size); + } + else if constexpr (size_type == 4) { + uint32_t size = item.size(); + write_wrapper(writer_, (char *)&size); } + else if constexpr (size_type == 8) { + uint64_t size = item.size(); + write_wrapper(writer_, (char *)&size); + } + else { + static_assert(!sizeof(item), "illegal size_type."); + } +#else else { + uint16_t size16; + uint32_t size32; + uint64_t size64; switch ((info.metainfo() & 0b11000) >> 3) { case 1: - writer_.write((char *)&size, 2); + size16 = item.size(); + write_wrapper<2>(writer_, (char *)&size16); break; case 2: - writer_.write((char *)&size, 4); + size32 = item.size(); + write_wrapper<4>(writer_, (char *)&size32); break; case 3: - writer_.write((char *)&size, 8); + size64 = item.size(); + write_wrapper<8>(writer_, (char *)&size64); break; default: unreachable(); @@ -204,12 +244,8 @@ class packer { #endif if constexpr (trivially_copyable_container) { using value_type = typename type::value_type; - auto container_size = 1ull * size * sizeof(value_type); - if SP_UNLIKELY (container_size >= PTRDIFF_MAX) - unreachable(); - else { - writer_.write((char *)item.data(), container_size); - } + write_wrapper(writer_, (char *)item.data(), + item.size()); return; } else { @@ -232,7 +268,7 @@ class packer { } else if constexpr (optional) { bool has_value = item.has_value(); - writer_.write((char *)&has_value, sizeof(bool)); + write_wrapper(writer_, (char *)&has_value); if (has_value) { serialize_one(*item); } @@ -241,7 +277,7 @@ class packer { static_assert(std::variant_size_v < 256, "variant's size is too large"); uint8_t index = item.index(); - writer_.write((char *)&index, sizeof(index)); + write_wrapper(writer_, (char *)&index); std::visit( [this](auto &&e) { this->serialize_one(e); @@ -250,7 +286,7 @@ class packer { } else if constexpr (expected) { bool has_value = item.has_value(); - writer_.write((char *)&has_value, sizeof(has_value)); + write_wrapper(writer_, (char *)&has_value); if (has_value) { if constexpr (!std::is_same_v) serialize_one(item.value()); @@ -266,10 +302,13 @@ class packer { std::is_aggregate_v>, "struct_pack only support aggregated type, or you should " "add macro STRUCT_PACK_REFL(Type,field1,field2...)"); - if constexpr (is_trivial_serializable::value) { - writer_.write((char *)&item, sizeof(type)); + if constexpr (is_trivial_serializable::value && + is_system_little_endian) { + write_wrapper(writer_, (char *)&item); } - else if constexpr (is_trivial_serializable::value) { + else if constexpr ((is_trivial_serializable::value && + !is_system_little_endian) || + is_trivial_serializable::value) { visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { int i = 1; ((serialize_one(items), @@ -291,7 +330,7 @@ class packer { if constexpr (id == type_id::compatible_t) { if constexpr (version == type::version_number) { bool has_value = item.has_value(); - writer_.write((char *)&has_value, sizeof(bool)); + write_wrapper(writer_, (char *)&has_value); if (has_value) { serialize_one(*item); } diff --git a/include/ylt/struct_pack/reflection.hpp b/include/ylt/struct_pack/reflection.hpp index 292efa77f..9847fa889 100644 --- a/include/ylt/struct_pack/reflection.hpp +++ b/include/ylt/struct_pack/reflection.hpp @@ -39,6 +39,9 @@ namespace struct_pack { namespace detail { +constexpr inline bool is_system_little_endian = + static_cast(1) == 1; + template using get_args_type = remove_cvref_t>, diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 83adebbbe..576ba3b47 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -34,6 +34,7 @@ #include "alignment.hpp" #include "calculate_size.hpp" #include "derived_helper.hpp" +#include "endian_wrapper.hpp" #include "error_code.hpp" #include "md5_constexpr.hpp" #include "packer.hpp" @@ -43,6 +44,7 @@ #include "type_id.hpp" #include "type_trait.hpp" #include "varint.hpp" +#include "ylt/struct_pack/util.h" namespace struct_pack { @@ -446,11 +448,30 @@ class unpacker { deserialize_compatible(unsigned compatible_sz_len) { constexpr std::size_t sz[] = {0, 2, 4, 8}; auto len_sz = sz[compatible_sz_len]; - uint64_t data_len = 0; - if SP_UNLIKELY (!reader_.read((char *)&data_len, len_sz)) { - return {errc::no_buffer_space, 0}; + uint64_t data_len64; + uint32_t data_len32; + uint16_t data_len16; + bool result; + switch (compatible_sz_len) { + case 1: + if SP_LIKELY (read_wrapper<2>(reader_, (char *)&data_len16)) { + return {errc{}, data_len16}; + } + break; + case 2: + if SP_LIKELY (read_wrapper<4>(reader_, (char *)&data_len32)) { + return {errc{}, data_len32}; + } + break; + case 3: + if SP_LIKELY (read_wrapper<8>(reader_, (char *)&data_len64)) { + return {errc{}, data_len64}; + } + break; + default: + unreachable(); } - return {errc{}, data_len}; + return {errc::no_buffer_space, 0}; } template @@ -467,7 +488,7 @@ class unpacker { } else { char buffer[literal.size() + 1]; - if SP_UNLIKELY (!reader_.read(buffer, literal.size() + 1)) { + if SP_UNLIKELY (!read_wrapper<1>(reader_, buffer, literal.size() + 1)) { return errc::no_buffer_space; } if SP_UNLIKELY (memcmp(buffer, literal.data(), literal.size() + 1)) { @@ -486,8 +507,8 @@ class unpacker { reader_.read_head((char *)¤t_types_code); } else { - if SP_UNLIKELY (!reader_.read((char *)¤t_types_code, - sizeof(uint32_t))) { + if SP_UNLIKELY (!read_wrapper( + reader_, (char *)¤t_types_code)) { return {struct_pack::errc::no_buffer_space, 0}; } constexpr uint32_t types_code = get_types_code(); @@ -501,7 +522,8 @@ class unpacker { return {}; } unsigned char metainfo; - if SP_UNLIKELY (!reader_.read((char *)&metainfo, sizeof(unsigned char))) { + if SP_UNLIKELY (!read_wrapper(reader_, + (char *)&metainfo)) { return {struct_pack::errc::no_buffer_space, 0}; } std::pair ret; @@ -558,6 +580,9 @@ class unpacker { static_assert(view_reader_t, "The Reader isn't a view_reader, can't deserialize " "a trivial_view"); + static_assert( + is_system_little_endian || sizeof(typename type::value_type) == 1, + "get a trivial view in big-endian system is limited."); const char *view = reader_.read_view(sizeof(typename T::value_type)); if SP_LIKELY (view != nullptr) { item = *reinterpret_cast(view); @@ -578,7 +603,7 @@ class unpacker { id == type_id::int128_t || id == type_id::uint128_t || id == type_id::bitset_t) { if constexpr (NotSkip) { - if SP_UNLIKELY (!reader_.read((char *)&item, sizeof(type))) { + if SP_UNLIKELY (!read_wrapper(reader_, (char *)&item)) { return struct_pack::errc::no_buffer_space; } } @@ -588,7 +613,8 @@ class unpacker { } else if constexpr (unique_ptr) { bool has_value{}; - if SP_UNLIKELY (!reader_.read((char *)&has_value, sizeof(bool))) { + if SP_UNLIKELY (!read_wrapper(reader_, + (char *)&has_value)) { return struct_pack::errc::no_buffer_space; } if (!has_value) { @@ -596,7 +622,7 @@ class unpacker { } if constexpr (is_base_class) { uint32_t id; - reader_.read((char *)&id, sizeof(id)); + read_wrapper(reader_, (char *)&id); bool ok; auto index = search_type_by_md5(id, ok); if SP_UNLIKELY (!ok) { @@ -622,7 +648,9 @@ class unpacker { else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value) { if constexpr (NotSkip) { - if SP_UNLIKELY (!reader_.read((char *)&item, sizeof(type))) { + if SP_UNLIKELY (!read_wrapper( + reader_, (char *)&item, + sizeof(item) / sizeof(item[0]))) { return struct_pack::errc::no_buffer_space; } } @@ -641,31 +669,55 @@ class unpacker { } } else if constexpr (container) { - size_t size = 0; + uint8_t size8; + uint16_t size16; + uint32_t size32; + uint64_t size64; + bool result; + if constexpr (size_type == 1) { + if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size8)) { + return struct_pack::errc::no_buffer_space; + } + size64 = size8; + } #ifdef STRUCT_PACK_OPTIMIZE - if SP_UNLIKELY (!reader_.read((char *)&size, size_type)) { - return struct_pack::errc::no_buffer_space; + else if constexpr (size_type == 2) { + if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size16)) { + return struct_pack::errc::no_buffer_space; + } + size64 = size16; } -#else - if constexpr (size_type == 1) { - if SP_UNLIKELY (!reader_.read((char *)&size, size_type)) { + else if constexpr (size_type == 4) { + if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size32)) { + return struct_pack::errc::no_buffer_space; + } + size64 = size32; + } + else if constexpr (size_type == 8) { + if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size64)) { return struct_pack::errc::no_buffer_space; } } + else { + static_assert(!sizeof(T), "illegal size_type"); + } +#else else { switch (size_type_) { case 1: - if SP_UNLIKELY (!reader_.read((char *)&size, 2)) { + if SP_UNLIKELY (!read_wrapper<2>(reader_, (char *)&size16)) { return struct_pack::errc::no_buffer_space; } + size64 = size16; break; case 2: - if SP_UNLIKELY (!reader_.read((char *)&size, 4)) { + if SP_UNLIKELY (!read_wrapper<4>(reader_, (char *)&size32)) { return struct_pack::errc::no_buffer_space; } + size64 = size32; break; case 3: - if SP_UNLIKELY (!reader_.read((char *)&size, 8)) { + if SP_UNLIKELY (!read_wrapper<8>(reader_, (char *)&size64)) { return struct_pack::errc::no_buffer_space; } break; @@ -674,7 +726,7 @@ class unpacker { } } #endif - if SP_UNLIKELY (size == 0) { + if SP_UNLIKELY (size64 == 0) { return {}; } if constexpr (map_container) { @@ -682,11 +734,12 @@ class unpacker { value{}; if constexpr (is_trivial_serializable::value && !NotSkip) { - return reader_.ignore(size * sizeof(value)) ? errc{} - : errc::no_buffer_space; + return reader_.ignore(size64 * sizeof(value)) + ? errc{} + : errc::no_buffer_space; } else { - for (size_t i = 0; i < size; ++i) { + for (uint64_t i = 0; i < size64; ++i) { code = deserialize_one(value); if SP_UNLIKELY (code != struct_pack::errc{}) { return code; @@ -702,11 +755,12 @@ class unpacker { typename type::value_type value{}; if constexpr (is_trivial_serializable::value && !NotSkip) { - return reader_.ignore(size * sizeof(value)) ? errc{} - : errc::no_buffer_space; + return reader_.ignore(size64 * sizeof(value)) + ? errc{} + : errc::no_buffer_space; } else { - for (size_t i = 0; i < size; ++i) { + for (uint64_t i = 0; i < size64; ++i) { code = deserialize_one(value); if SP_UNLIKELY (code != struct_pack::errc{}) { return code; @@ -721,25 +775,29 @@ class unpacker { else { using value_type = typename type::value_type; if constexpr (trivially_copyable_container) { - size_t mem_sz = size * sizeof(value_type); + uint64_t mem_sz = size64 * sizeof(value_type); if constexpr (NotSkip) { if constexpr (string_view || dynamic_span) { static_assert( view_reader_t, "The Reader isn't a view_reader, can't deserialize " "a string_view/span"); + static_assert( + sizeof(value_type) == 1 || is_system_little_endian, + "zero-copy in big endian is limit."); const char *view = reader_.read_view(mem_sz); if SP_UNLIKELY (view == nullptr) { return struct_pack::errc::no_buffer_space; } - item = {(value_type *)(view), size}; + item = {(value_type *)(view), size64}; } else { if SP_UNLIKELY (mem_sz >= PTRDIFF_MAX) unreachable(); else { - item.resize(size); - if SP_UNLIKELY (!reader_.read((char *)item.data(), mem_sz)) { + item.resize(size64); + if SP_UNLIKELY (!read_wrapper( + reader_, (char *)item.data(), size64)) { return struct_pack::errc::no_buffer_space; } } @@ -757,7 +815,7 @@ class unpacker { "is a non-trival-serializable type."); } else { - item.resize(size); + item.resize(size64); for (auto &i : item) { code = deserialize_one(i); if SP_UNLIKELY (code != struct_pack::errc{}) { @@ -768,7 +826,7 @@ class unpacker { } else { value_type useless; - for (size_t i = 0; i < size; ++i) { + for (size_t i = 0; i < size64; ++i) { code = deserialize_one(useless); if SP_UNLIKELY (code != struct_pack::errc{}) { return code; @@ -792,7 +850,8 @@ class unpacker { } else if constexpr (optional || expected) { bool has_value{}; - if SP_UNLIKELY (!reader_.read((char *)&has_value, sizeof(bool))) { + if SP_UNLIKELY (!read_wrapper(reader_, + (char *)&has_value)) { return struct_pack::errc::no_buffer_space; } if SP_UNLIKELY (!has_value) { @@ -817,7 +876,7 @@ class unpacker { } else if constexpr (is_variant_v) { uint8_t index{}; - if SP_UNLIKELY (!reader_.read((char *)&index, sizeof(index))) { + if SP_UNLIKELY (!read_wrapper(reader_, (char *)&index)) { return struct_pack::errc::no_buffer_space; } if SP_UNLIKELY (index >= std::variant_size_v) { @@ -837,9 +896,11 @@ class unpacker { std::is_aggregate_v>, "struct_pack only support aggregated type, or you should " "add macro STRUCT_PACK_REFL(Type,field1,field2...)"); - if constexpr (is_trivial_serializable::value) { + if constexpr (is_trivial_serializable::value && + is_system_little_endian) { if constexpr (NotSkip) { - if SP_UNLIKELY (!reader_.read((char *)&item, sizeof(type))) { + if SP_UNLIKELY (!read_wrapper(reader_, + (char *)&item)) { return struct_pack::errc::no_buffer_space; } } @@ -848,7 +909,9 @@ class unpacker { : errc::no_buffer_space; } } - else if constexpr (is_trivial_serializable::value) { + else if constexpr ((is_trivial_serializable::value && + !is_system_little_endian) || + is_trivial_serializable::value) { visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { int i = 1; auto f = [&](auto &&item) { @@ -883,7 +946,8 @@ class unpacker { return struct_pack::errc::no_buffer_space; } bool has_value{}; - if SP_UNLIKELY (!reader_.read((char *)&has_value, sizeof(bool))) { + if SP_UNLIKELY (!read_wrapper(reader_, + (char *)&has_value)) { return struct_pack::errc::no_buffer_space; } if (!has_value) { @@ -1062,7 +1126,8 @@ class unpacker { template struct MD5_reader_wrapper : public Reader { MD5_reader_wrapper(Reader &&reader) : Reader(std::move(reader)) { - is_failed = !Reader::read((char *)&head_chars, sizeof(head_chars)); + is_failed = + !read_wrapper(*(Reader *)this, (char *)&head_chars); } bool read_head(char *target) { if SP_UNLIKELY (is_failed) { diff --git a/include/ylt/struct_pack/varint.hpp b/include/ylt/struct_pack/varint.hpp index 799106e36..701a127d7 100644 --- a/include/ylt/struct_pack/varint.hpp +++ b/include/ylt/struct_pack/varint.hpp @@ -208,10 +208,10 @@ STRUCT_PACK_INLINE void serialize_varint(writer& writer_, const T& t) { } while (v >= 0x80) { uint8_t temp = v | 0x80u; - writer_.write((char*)&temp, sizeof(temp)); + write_wrapper(writer_, (char*)&temp); v >>= 7; } - writer_.write((char*)&v, sizeof(char)); + write_wrapper(writer_, (char*)&v); } #if __cpp_concepts >= 201907L template diff --git a/src/struct_pack/tests/binary_data/test_cross_platform.dat b/src/struct_pack/tests/binary_data/test_cross_platform.dat index 0338937e82b3d47ab52d68fe0a942d1e63fcdd6e..f5a6d0a3fe6d9d604e9d88ac905322bacdd0ed2e 100644 GIT binary patch delta 124 zcmey*_=<^TsUef@L>5=Bmd3w~LV}&0|C=XzZcAkPbVi5)1dPGpxBaY{k`5rgv;l}@ vU`PXlZ}w?vY7QVi3y1&$i1;&mW8=;CAilP;Acz8+vw8DT2Q_haFAyI9f&LzD delta 15 XcmaFG^q-L>`cmMxi7c)YgSPQdmhi7Z(ifs, object); + } + { + std::ofstream ifs("binary_data/test_cross_platform_without_debug_info.dat"); + auto object = create_complicated_object(); + serialize_to(ifs, object); + } +} + TEST_CASE("testing deserialize other platform data") { std::ifstream ifs("binary_data/test_cross_platform.dat"); if (!ifs.is_open()) { diff --git a/src/struct_pack/tests/test_struct.hpp b/src/struct_pack/tests/test_struct.hpp index 38b29d6b2..a66218a37 100644 --- a/src/struct_pack/tests/test_struct.hpp +++ b/src/struct_pack/tests/test_struct.hpp @@ -40,6 +40,15 @@ struct empty {}; enum class Color { red, black, white }; +struct trivial_one { + int a; + double b; + float c; + auto operator==(const trivial_one &rhs) const { + return a == rhs.a && b == rhs.b && c == rhs.c; + } +}; + struct complicated_object { Color color; int a; @@ -56,30 +65,34 @@ struct complicated_object { std::array m; person n[2]; std::pair o; + std::vector> p; bool operator==(const complicated_object &o) const { return color == o.color && a == o.a && b == o.b && c == o.c && d == o.d && e == o.e && f == o.f && g == o.g && h == o.h && i == o.i && j == o.j && k == o.k && m == o.m && n[0] == o.n[0] && - n[1] == o.n[1] && this->o == o.o; + n[1] == o.n[1] && this->o == o.o && p == o.p; } }; inline complicated_object create_complicated_object() { - return complicated_object{Color::red, - 42, - "hello", - {{20, "tom"}, {22, "jerry"}}, - {"hello", "world"}, - {1, 2}, - {{1, {20, "tom"}}}, - {{1, {20, "tom"}}, {1, {22, "jerry"}}}, - {"aa", "bb"}, - {1, 2}, - {{1, {20, "tom"}}}, - {{1, 2}}, - {person{20, "tom"}, {22, "jerry"}}, - {person{15, "tom"}, {31, "jerry"}}, - std::make_pair("aa", person{20, "tom"})}; + return complicated_object{ + Color::red, + 42, + "hello", + {{20, "tom"}, {22, "jerry"}}, + {"hello", "world"}, + {1, 2}, + {{1, {20, "tom"}}}, + {{1, {20, "tom"}}, {1, {22, "jerry"}}}, + {"aa", "bb"}, + {1, 2}, + {{1, {20, "tom"}}}, + {{1, 2}}, + {person{20, "tom"}, {22, "jerry"}}, + {person{15, "tom"}, {31, "jerry"}}, + std::make_pair("aa", person{20, "tom"}), + {{trivial_one{1232114, 1.7, 2.4}, trivial_one{12315, 1.4, 2.6}}, + {trivial_one{4, 0.7, 1.4}, trivial_one{1123115, 11111.4, 2213321.6}}}}; } struct nested_object { From 34f9b3dd7baf8e09d3a1e2ee8eee0591ca4fcefa Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 16:36:14 +0800 Subject: [PATCH 02/32] fix --- include/ylt/struct_pack/varint.hpp | 1 + src/struct_pack/tests/test_cross_platform.cpp | 32 ------------------- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/include/ylt/struct_pack/varint.hpp b/include/ylt/struct_pack/varint.hpp index 701a127d7..d5689cc60 100644 --- a/include/ylt/struct_pack/varint.hpp +++ b/include/ylt/struct_pack/varint.hpp @@ -19,6 +19,7 @@ #include #include +#include "endian_wrapper.hpp" #include "reflection.hpp" namespace struct_pack { diff --git a/src/struct_pack/tests/test_cross_platform.cpp b/src/struct_pack/tests/test_cross_platform.cpp index a320cdf8b..0e0df1fe9 100644 --- a/src/struct_pack/tests/test_cross_platform.cpp +++ b/src/struct_pack/tests/test_cross_platform.cpp @@ -44,36 +44,4 @@ TEST_CASE("testing deserialize other platform data without debug info") { REQUIRE(result.has_value()); auto object = create_complicated_object(); CHECK(result.value() == object); -} - -TEST_CASE("testing serialize other platform data") { - std::ifstream ifs("binary_data/test_cross_platform.dat"); - if (!ifs.is_open()) { - ifs.open("src/struct_pack/tests/binary_data/test_cross_platform.dat"); - } - REQUIRE(ifs.is_open()); - std::string content(std::istreambuf_iterator{ifs}, - std::istreambuf_iterator{}); - auto object = create_complicated_object(); - auto buffer = - struct_pack::serialize(object); - CHECK(buffer == content); -} - -TEST_CASE("testing serialize other platform data") { - std::ifstream ifs("binary_data/test_cross_platform_without_debug_info.dat"); - if (!ifs.is_open()) { - ifs.open( - "src/struct_pack/tests/binary_data/" - "test_cross_platform_without_debug_info.dat"); - } - REQUIRE(ifs.is_open()); - std::string content(std::istreambuf_iterator{ifs}, - std::istreambuf_iterator{}); - auto object = create_complicated_object(); - auto buffer = - struct_pack::serialize(object); - CHECK(buffer == content); } \ No newline at end of file From 6c6172a93c9d120f58ca0d5217b1806c5876925e Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 16:47:16 +0800 Subject: [PATCH 03/32] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 8 ++++---- include/ylt/struct_pack/packer.hpp | 2 +- include/ylt/struct_pack/unpacker.hpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 6797ab262..450881263 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -18,7 +18,7 @@ #include "ylt/struct_pack/util.h" namespace struct_pack::detail { -template +template void write_wrapper(writer_t& writer, const char* data) { if constexpr (is_system_little_endian || block_size == 1) { writer.write(data, block_size); @@ -99,7 +99,7 @@ void write_wrapper(writer_t& writer, const char* data) { static_assert(sizeof(writer), "illegal block size(should be 1,2,4,8,16)"); } } -template +template void write_wrapper(writer_t& writer, const char* data, std::size_t block_count) { if constexpr (is_system_little_endian || block_size == 1) { @@ -116,7 +116,7 @@ void write_wrapper(writer_t& writer, const char* data, } } } -template +template bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { if constexpr (is_system_little_endian || block_size == 1) { return static_cast(reader.read(data, block_size)); @@ -196,7 +196,7 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { } } } -template +template bool read_wrapper(reader_t& reader, char* SP_RESTRICT data, std::size_t block_count) { if constexpr (is_system_little_endian || block_size == 1) { diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index eafce4ba3..50e6b1ee4 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -17,8 +17,8 @@ #include #include "calculate_size.hpp" -#include "reflection.hpp" #include "endian_wrapper.hpp" +#include "reflection.hpp" #include "ylt/struct_pack/util.h" namespace struct_pack::detail { template < diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 576ba3b47..9c85758a5 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -789,7 +789,7 @@ class unpacker { if SP_UNLIKELY (view == nullptr) { return struct_pack::errc::no_buffer_space; } - item = {(value_type *)(view), size64}; + item = {(value_type *)(view), (std::size_t)size64}; } else { if SP_UNLIKELY (mem_sz >= PTRDIFF_MAX) From 8654aa12a552c709c7e3209d4b21fc3db4a39df4 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:12:26 +0800 Subject: [PATCH 04/32] add ci in s390x --- .github/workflows/s390x.yml | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/s390x.yml diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml new file mode 100644 index 000000000..184a4d3c9 --- /dev/null +++ b/.github/workflows/s390x.yml @@ -0,0 +1,45 @@ +name: IBM S390X + +on: + workflow_call: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + workflow_dispatch: + +jobs: + build-ubuntu-s390x: + name: Build Linux on s390x arch and run unit tests + runs-on: ubuntu-latest + strategy: + matrix: + mode: [ Debug, Release ] #[Release, Debug] + runs-on: ubuntu-22.04 + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install ninja-build tool + uses: seanmiddleditch/gha-setup-ninja@v3 + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ github.job }}-${{ matrix.mode}} + + - name: Configure + run: | + CXX=g++ CC=gcc + cmake -B ${{github.workspace}}/build -G Ninja \ + -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ + -DUSE_CCACHE=${{env.ccache}} \ + -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF + + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} + + - name: Test + working-directory: ${{github.workspace}}/build + run: ctest -C ${{matrix.mode}} -j 1 -V \ No newline at end of file From 68a63960a51ada86085bfea8c59303d9c3289e4f Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:17:20 +0800 Subject: [PATCH 05/32] fix --- .github/workflows/s390x.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 184a4d3c9..4993abf29 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -10,8 +10,6 @@ on: jobs: build-ubuntu-s390x: - name: Build Linux on s390x arch and run unit tests - runs-on: ubuntu-latest strategy: matrix: mode: [ Debug, Release ] #[Release, Debug] From dd195e16dbf9d5ea99e0d2b698f9cfecc6b0fd35 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:18:59 +0800 Subject: [PATCH 06/32] fix --- .github/workflows/s390x.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 4993abf29..7b5230757 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -34,6 +34,7 @@ jobs: -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ -DUSE_CCACHE=${{env.ccache}} \ -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF + -DBUILD_STRUCT_PB=OFF - name: Build run: cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} From 0cc36b5d60d2bdf1f01d8a761befc52045acd11b Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:22:37 +0800 Subject: [PATCH 07/32] fix --- .github/workflows/s390x.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 7b5230757..4c3d6cbac 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -28,6 +28,8 @@ jobs: key: ${{ github.job }}-${{ matrix.mode}} - name: Configure + with: + arch: s390x run: | CXX=g++ CC=gcc cmake -B ${{github.workspace}}/build -G Ninja \ @@ -37,8 +39,12 @@ jobs: -DBUILD_STRUCT_PB=OFF - name: Build + with: + arch: s390x run: cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} - name: Test + with: + arch: s390x working-directory: ${{github.workspace}}/build run: ctest -C ${{matrix.mode}} -j 1 -V \ No newline at end of file From f667c57e7ab5828ba1b2d218addef6584d0a09c4 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:25:25 +0800 Subject: [PATCH 08/32] fix --- .github/workflows/s390x.yml | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 4c3d6cbac..5cfe29f4e 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -30,21 +30,19 @@ jobs: - name: Configure with: arch: s390x - run: | - CXX=g++ CC=gcc - cmake -B ${{github.workspace}}/build -G Ninja \ - -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ - -DUSE_CCACHE=${{env.ccache}} \ - -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF - -DBUILD_STRUCT_PB=OFF - - - name: Build - with: - arch: s390x - run: cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} - - - name: Test - with: - arch: s390x - working-directory: ${{github.workspace}}/build - run: ctest -C ${{matrix.mode}} -j 1 -V \ No newline at end of file + distro: ubuntu-22.04 + install: | + apt-get update -q -y + apt-get -y install cmake + apt-get -y install make + apt-get -y install ninja + apt-get -y install g++ + run: | + CXX=g++ CC=gcc + cmake -B ${{github.workspace}}/build -G Ninja \ + -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ + -DUSE_CCACHE=${{env.ccache}} \ + -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF + -DBUILD_STRUCT_PB=OFF + cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} + ctest -C ${{matrix.mode}} -j 1 -V \ No newline at end of file From 2c05c20b48b6be9208c28fbac881c55ecf0f5314 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:27:57 +0800 Subject: [PATCH 09/32] fix --- .github/workflows/s390x.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 5cfe29f4e..c0279fc0f 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -12,22 +12,14 @@ jobs: build-ubuntu-s390x: strategy: matrix: - mode: [ Debug, Release ] #[Release, Debug] + mode: [ Debug, Release ] #[Release, Debug] runs-on: ubuntu-22.04 steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install ninja-build tool - uses: seanmiddleditch/gha-setup-ninja@v3 - - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ github.job }}-${{ matrix.mode}} - - - name: Configure + - uses: actions/checkout@v3 + - uses: uraimo/run-on-arch-action@v2 + - name: Test + id: runcmd with: arch: s390x distro: ubuntu-22.04 @@ -38,6 +30,7 @@ jobs: apt-get -y install ninja apt-get -y install g++ run: | + lscpu | grep Endian CXX=g++ CC=gcc cmake -B ${{github.workspace}}/build -G Ninja \ -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ From c7297df8f270d4eaf2a2542c405abe18a29c87f4 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:30:50 +0800 Subject: [PATCH 10/32] fix --- .github/workflows/s390x.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index c0279fc0f..9914fa105 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -1,7 +1,6 @@ name: IBM S390X on: - workflow_call: push: branches: [ main, develop ] pull_request: @@ -10,15 +9,12 @@ on: jobs: build-ubuntu-s390x: - strategy: - matrix: - mode: [ Debug, Release ] #[Release, Debug] + name: Build Linux on s390x arch and run unit tests runs-on: ubuntu-22.04 - steps: - uses: actions/checkout@v3 - uses: uraimo/run-on-arch-action@v2 - - name: Test + name: Test id: runcmd with: arch: s390x @@ -27,12 +23,11 @@ jobs: apt-get update -q -y apt-get -y install cmake apt-get -y install make - apt-get -y install ninja apt-get -y install g++ run: | lscpu | grep Endian CXX=g++ CC=gcc - cmake -B ${{github.workspace}}/build -G Ninja \ + cmake -B ${{github.workspace}}/build \ -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ -DUSE_CCACHE=${{env.ccache}} \ -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF From e0ee1ea2a6eb170fe66c3740154e6bc23224ee09 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:34:37 +0800 Subject: [PATCH 11/32] fix --- .github/workflows/s390x.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 9914fa105..7c4ece94f 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -10,7 +10,7 @@ on: jobs: build-ubuntu-s390x: name: Build Linux on s390x arch and run unit tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: uraimo/run-on-arch-action@v2 @@ -18,7 +18,7 @@ jobs: id: runcmd with: arch: s390x - distro: ubuntu-22.04 + distro: ubuntu-latest install: | apt-get update -q -y apt-get -y install cmake From 1a53fd8050af65674d29fb2e37d9f4810a6a1ccc Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:43:11 +0800 Subject: [PATCH 12/32] fix --- .github/workflows/s390x.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 7c4ece94f..00e882760 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -18,7 +18,8 @@ jobs: id: runcmd with: arch: s390x - distro: ubuntu-latest + distro: ubuntu22.04 + githubToken: ${{ github.token }} install: | apt-get update -q -y apt-get -y install cmake From f4547602007251037bc25bd05c1cf79b3d55beca Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:51:04 +0800 Subject: [PATCH 13/32] fix --- .github/workflows/s390x.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 00e882760..3fe7cb1e5 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -29,9 +29,7 @@ jobs: lscpu | grep Endian CXX=g++ CC=gcc cmake -B ${{github.workspace}}/build \ - -DCMAKE_BUILD_TYPE=${{matrix.mode}} \ - -DUSE_CCACHE=${{env.ccache}} \ - -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF - -DBUILD_STRUCT_PB=OFF + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF -DBUILD_STRUCT_PB=OFF cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} ctest -C ${{matrix.mode}} -j 1 -V \ No newline at end of file From 44afd2c24194e27988b24b730bd899315717c841 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 17:54:57 +0800 Subject: [PATCH 14/32] fix --- .github/workflows/s390x.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 3fe7cb1e5..5b9a60e4c 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -31,5 +31,5 @@ jobs: cmake -B ${{github.workspace}}/build \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF -DBUILD_STRUCT_PB=OFF - cmake --build ${{github.workspace}}/build --config ${{matrix.mode}} - ctest -C ${{matrix.mode}} -j 1 -V \ No newline at end of file + cmake --build ${{github.workspace}}/build + ctest -j 1 -V \ No newline at end of file From fb036fc170b30139d92424866b6c4ffaf1d70973 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Wed, 11 Oct 2023 18:26:52 +0800 Subject: [PATCH 15/32] fix --- .github/workflows/s390x.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 5b9a60e4c..b2e13a602 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -31,5 +31,6 @@ jobs: cmake -B ${{github.workspace}}/build \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF -DBUILD_STRUCT_PB=OFF - cmake --build ${{github.workspace}}/build + cmake --build ${{github.workspace}}/build -j + cd ${{github.workspace}}/build ctest -j 1 -V \ No newline at end of file From 7e69e272b813f0acb171740368655ee42de598de Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 10:42:09 +0800 Subject: [PATCH 16/32] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 4 ++-- include/ylt/struct_pack/reflection.hpp | 7 +++++-- src/struct_pack/tests/test_cross_platform.cpp | 5 +++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 450881263..b1867657e 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -40,7 +40,7 @@ void write_wrapper(writer_t& writer, const char* data) { auto tmp = _byteswap_ulong(*data); writer.write(&tmp, block_size); #elif defined(__clang__) || defined(__GNUC__) - auto tmp = __builtin_bswap16(*data); + auto tmp = __builtin_bswap32(*data); writer.write(&tmp, block_size); #else writer.write(data + 3, 1); @@ -54,7 +54,7 @@ void write_wrapper(writer_t& writer, const char* data) { auto tmp = _byteswap_uint64(*data); writer.write(&tmp, block_size); #elif defined(__clang__) || defined(__GNUC__) - auto tmp = __builtin_bswap16(*data); + auto tmp = __builtin_bswap64(*data); writer.write(&tmp, block_size); #else writer.write(data + 7, 1); diff --git a/include/ylt/struct_pack/reflection.hpp b/include/ylt/struct_pack/reflection.hpp index 9847fa889..3c029645b 100644 --- a/include/ylt/struct_pack/reflection.hpp +++ b/include/ylt/struct_pack/reflection.hpp @@ -39,8 +39,11 @@ namespace struct_pack { namespace detail { -constexpr inline bool is_system_little_endian = - static_cast(1) == 1; +constexpr inline bool is_system_little_endian_impl(int value = 0x01) { + return static_cast(value) == 1; +} + +constexpr inline bool is_system_little_endian = is_system_little_endian_impl(); template using get_args_type = remove_cvref_t #include +#include #include #include "doctest.h" #include "test_struct.hpp" +#include "ylt/struct_pack/reflection.hpp" using namespace struct_pack; using namespace doctest; @@ -21,6 +23,9 @@ void data_gen() { } TEST_CASE("testing deserialize other platform data") { + std::cout << "Now endian:" + << (struct_pack::detail::is_system_little_endian ? "little" : "big") + << std::endl; std::ifstream ifs("binary_data/test_cross_platform.dat"); if (!ifs.is_open()) { ifs.open("src/struct_pack/tests/binary_data/test_cross_platform.dat"); From b9dc375f227c800000f54a7a3df85c4c7c952ce3 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 11:09:15 +0800 Subject: [PATCH 17/32] fix --- .github/workflows/s390x.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index b2e13a602..8b9803645 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -21,7 +21,6 @@ jobs: distro: ubuntu22.04 githubToken: ${{ github.token }} install: | - apt-get update -q -y apt-get -y install cmake apt-get -y install make apt-get -y install g++ @@ -32,5 +31,5 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF -DBUILD_STRUCT_PB=OFF cmake --build ${{github.workspace}}/build -j - cd ${{github.workspace}}/build - ctest -j 1 -V \ No newline at end of file + cd ${{github.workspace}}/build/output/tests + ./struct_pack_test \ No newline at end of file From 865ee361aea5f6ec549f31a5246d9047fdc8f715 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 11:11:52 +0800 Subject: [PATCH 18/32] fix --- .github/workflows/s390x.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 8b9803645..7cea7b341 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -21,6 +21,7 @@ jobs: distro: ubuntu22.04 githubToken: ${{ github.token }} install: | + apt-get update -q -y apt-get -y install cmake apt-get -y install make apt-get -y install g++ From 3c878439334bb7b9413069976a3a7119944f8929 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 11:44:45 +0800 Subject: [PATCH 19/32] fix endian checker --- include/ylt/struct_pack/endian_wrapper.hpp | 64 ++++++++++++++++++++++ include/ylt/struct_pack/reflection.hpp | 4 -- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index b1867657e..e2543ba1a 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -14,9 +14,73 @@ * limitations under the License. */ #pragma once +#include + #include "reflection.hpp" #include "ylt/struct_pack/util.h" + namespace struct_pack::detail { +#if __cpp_lib_endian >= 201907L +constexpr inline bool is_system_little_endian = + (std::endian::little == std::endian::native); +static_assert(std::endian::native == std::endian::little || + std::endian::native == std::endian::big, + "struct_pack don't support middle-endian"); +#else +#define BYTEORDER_LITTLE_ENDIAN 0 // Little endian machine. +#define BYTEORDER_BIG_ENDIAN 1 // Big endian machine. + +//#define BYTEORDER_ENDIAN BYTEORDER_LITTLE_ENDIAN + +#ifndef BYTEORDER_ENDIAN +// Detect with GCC 4.6's macro. +#if defined(__BYTE_ORDER__) +#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#define BYTEORDER_ENDIAN BYTEORDER_LITTLE_ENDIAN +#elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#define BYTEORDER_ENDIAN BYTEORDER_BIG_ENDIAN +#else +#error \ + "Unknown machine byteorder endianness detected. User needs to define BYTEORDER_ENDIAN." +#endif +// Detect with GLIBC's endian.h. +#elif defined(__GLIBC__) +#include +#if (__BYTE_ORDER == __LITTLE_ENDIAN) +#define BYTEORDER_ENDIAN BYTEORDER_LITTLE_ENDIAN +#elif (__BYTE_ORDER == __BIG_ENDIAN) +#define BYTEORDER_ENDIAN BYTEORDER_BIG_ENDIAN +#else +#error \ + "Unknown machine byteorder endianness detected. User needs to define BYTEORDER_ENDIAN." +#endif +// Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro. +#elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) +#define BYTEORDER_ENDIAN BYTEORDER_LITTLE_ENDIAN +#elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) +#define BYTEORDER_ENDIAN BYTEORDER_BIG_ENDIAN +// Detect with architecture macros. +#elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || \ + defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || \ + defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || \ + defined(__s390__) +#define BYTEORDER_ENDIAN BYTEORDER_BIG_ENDIAN +#elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || \ + defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || \ + defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || \ + defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_X64) || defined(__bfin__) +#define BYTEORDER_ENDIAN BYTEORDER_LITTLE_ENDIAN +#elif defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64)) +#define BYTEORDER_ENDIAN BYTEORDER_LITTLE_ENDIAN +#else +#error \ + "Unknown machine byteorder endianness detected. User needs to define BYTEORDER_ENDIAN." +#endif +#endif +constexpr inline bool is_system_little_endian = + (BYTEORDER_ENDIAN == BYTEORDER_LITTLE_ENDIAN); +#endif template void write_wrapper(writer_t& writer, const char* data) { diff --git a/include/ylt/struct_pack/reflection.hpp b/include/ylt/struct_pack/reflection.hpp index 3c029645b..f5cb38c08 100644 --- a/include/ylt/struct_pack/reflection.hpp +++ b/include/ylt/struct_pack/reflection.hpp @@ -39,11 +39,7 @@ namespace struct_pack { namespace detail { -constexpr inline bool is_system_little_endian_impl(int value = 0x01) { - return static_cast(value) == 1; -} -constexpr inline bool is_system_little_endian = is_system_little_endian_impl(); template using get_args_type = remove_cvref_t Date: Thu, 12 Oct 2023 15:06:36 +0800 Subject: [PATCH 20/32] fix compile error in big endian --- cmake/build.cmake | 10 ++- include/ylt/struct_pack/endian_wrapper.hpp | 34 ++++---- include/ylt/struct_pack/packer.hpp | 21 +++-- include/ylt/struct_pack/unpacker.hpp | 43 +++++++--- src/struct_pack/tests/CMakeLists.txt | 6 ++ src/struct_pack/tests/test_data_struct2.cpp | 87 +++------------------ src/struct_pack/tests/test_serialize.cpp | 18 ++++- 7 files changed, 105 insertions(+), 114 deletions(-) diff --git a/cmake/build.cmake b/cmake/build.cmake index 4e81f1119..aed55333b 100644 --- a/cmake/build.cmake +++ b/cmake/build.cmake @@ -1,4 +1,4 @@ -message(STATUS "-------------COMPILE Setting-------------") +message(STATUS "-------------COMPILE SETTING-------------") # CPP Standard foreach(i ${CMAKE_CXX_COMPILE_FEATURES}) @@ -31,6 +31,14 @@ if(BUILD_WITH_LIBCXX AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") else() endif() +include (TestBigEndian) +TEST_BIG_ENDIAN(IS_BIG_ENDIAN) +if(IS_BIG_ENDIAN) + message(STATUS "ENDIAN: BIG") +else() + message(STATUS "ENDIAN: LITTLE") +endif() + # force use lld if your compiler is clang # When using coro_rpc_client to send request, only remote function declarations are required. diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index e2543ba1a..7a5e87169 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -82,6 +82,10 @@ constexpr inline bool is_system_little_endian = (BYTEORDER_ENDIAN == BYTEORDER_LITTLE_ENDIAN); #endif +template +constexpr inline bool is_little_endian_copyable = + is_system_little_endian || block_size == 1; + template void write_wrapper(writer_t& writer, const char* data) { if constexpr (is_system_little_endian || block_size == 1) { @@ -93,7 +97,7 @@ void write_wrapper(writer_t& writer, const char* data) { writer.write(&tmp, block_size); #elif defined(__clang__) || defined(__GNUC__) auto tmp = __builtin_bswap16(*data); - writer.write(&tmp, block_size); + writer.write((char*)&tmp, block_size); #else writer.write(data + 1, 1); writer.write(data, 1); @@ -105,7 +109,7 @@ void write_wrapper(writer_t& writer, const char* data) { writer.write(&tmp, block_size); #elif defined(__clang__) || defined(__GNUC__) auto tmp = __builtin_bswap32(*data); - writer.write(&tmp, block_size); + writer.write((char*)&tmp, block_size); #else writer.write(data + 3, 1); writer.write(data + 2, 1); @@ -119,7 +123,7 @@ void write_wrapper(writer_t& writer, const char* data) { writer.write(&tmp, block_size); #elif defined(__clang__) || defined(__GNUC__) auto tmp = __builtin_bswap64(*data); - writer.write(&tmp, block_size); + writer.write((char*)&tmp, block_size); #else writer.write(data + 7, 1); writer.write(data + 6, 1); @@ -138,8 +142,8 @@ void write_wrapper(writer_t& writer, const char* data) { writer.write(&tmp1, block_size); #elif defined(__clang__) || defined(__GNUC__) auto tmp1 = __builtin_bswap64(*data), tmp2 = __builtin_bswap64(*data + 8); - writer.write(&tmp2, block_size); - writer.write(&tmp1, block_size); + writer.write((char*)&tmp2, block_size); + writer.write((char*)&tmp1, block_size); #else writer.write(data + 15, 1); writer.write(data + 14, 1); @@ -160,7 +164,7 @@ void write_wrapper(writer_t& writer, const char* data) { #endif } else { - static_assert(sizeof(writer), "illegal block size(should be 1,2,4,8,16)"); + static_assert(!sizeof(writer), "illegal block size(should be 1,2,4,8,16)"); } } template @@ -174,10 +178,7 @@ void write_wrapper(writer_t& writer, const char* data, writer.write(data, total_sz); } else { - for (std::size_t i = 0; i < block_count; ++i) { - write_wrapper(writer, data); - data += block_size; - } + static_assert(!sizeof(writer_t), "illegal use"); } } template @@ -187,7 +188,7 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { } else { std::array tmp; - bool res = static_cast(reader.read(&tmp, block_size)); + bool res = static_cast(reader.read((char*)&tmp, block_size)); if SP_UNLIKELY (res) { return res; } @@ -256,8 +257,10 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { #endif } else { - static_assert(sizeof(reader), "illegal block size(should be 1,2,4,8,16)"); + static_assert(!sizeof(reader), + "illegal block size(should be 1,2,4,8,16)"); } + return true; } } template @@ -271,12 +274,7 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data, return static_cast(reader.read(data, total_sz)); } else { - for (std::size_t i = 0; i < block_count; ++i) { - if SP_UNLIKELY (!read_wrapper(reader, data)) { - return false; - } - data += block_size; - } + static_assert(!sizeof(reader_t), "illegal use"); } } }; // namespace struct_pack::detail \ No newline at end of file diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 50e6b1ee4..6cdf7c52c 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -19,6 +19,7 @@ #include "calculate_size.hpp" #include "endian_wrapper.hpp" #include "reflection.hpp" +#include "ylt/struct_pack/type_id.hpp" #include "ylt/struct_pack/util.h" namespace struct_pack::detail { template < @@ -158,10 +159,12 @@ class packer { // do nothing } else if constexpr (std::is_fundamental_v || std::is_enum_v || - id == type_id::int128_t || id == type_id::uint128_t || - id == type_id::bitset_t) { + id == type_id::int128_t || id == type_id::uint128_t) { write_wrapper(writer_, (char *)&item); } + else if constexpr (id == type_id::bitset_t) { + write_wrapper(writer_, (char *)&item, sizeof(item)); + } else if constexpr (unique_ptr) { bool has_value = (item != nullptr); write_wrapper(writer_, (char *)&has_value); @@ -188,9 +191,10 @@ class packer { detail::serialize_varint(writer_, item); } else if constexpr (id == type_id::array_t) { - if constexpr (is_trivial_serializable::value) { - write_wrapper( - writer_, (char *)&item, sizeof(type) / sizeof(decltype(item[0]))); + if constexpr (is_trivial_serializable::value && + is_little_endian_copyable) { + write_wrapper(writer_, (char *)&item, + sizeof(type) / sizeof(item[0])); } else { for (const auto &i : item) { @@ -242,7 +246,8 @@ class packer { } } #endif - if constexpr (trivially_copyable_container) { + if constexpr (trivially_copyable_container && + is_little_endian_copyable) { using value_type = typename type::value_type; write_wrapper(writer_, (char *)item.data(), item.size()); @@ -303,11 +308,11 @@ class packer { "struct_pack only support aggregated type, or you should " "add macro STRUCT_PACK_REFL(Type,field1,field2...)"); if constexpr (is_trivial_serializable::value && - is_system_little_endian) { + is_little_endian_copyable) { write_wrapper(writer_, (char *)&item); } else if constexpr ((is_trivial_serializable::value && - !is_system_little_endian) || + !is_little_endian_copyable) || is_trivial_serializable::value) { visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { int i = 1; diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 9c85758a5..859c5fc3d 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -581,8 +581,9 @@ class unpacker { "The Reader isn't a view_reader, can't deserialize " "a trivial_view"); static_assert( - is_system_little_endian || sizeof(typename type::value_type) == 1, - "get a trivial view in big-endian system is limited."); + is_little_endian_copyable, + "get a trivial view with byte width > 1 in big-endian system is " + "not allowed."); const char *view = reader_.read_view(sizeof(typename T::value_type)); if SP_LIKELY (view != nullptr) { item = *reinterpret_cast(view); @@ -600,8 +601,7 @@ class unpacker { // do nothing } else if constexpr (std::is_fundamental_v || std::is_enum_v || - id == type_id::int128_t || id == type_id::uint128_t || - id == type_id::bitset_t) { + id == type_id::int128_t || id == type_id::uint128_t) { if constexpr (NotSkip) { if SP_UNLIKELY (!read_wrapper(reader_, (char *)&item)) { return struct_pack::errc::no_buffer_space; @@ -611,6 +611,17 @@ class unpacker { return reader_.ignore(sizeof(type)) ? errc{} : errc::no_buffer_space; } } + else if constexpr (id == type_id::bitset_t) { + if constexpr (NotSkip) { + if SP_UNLIKELY (!read_wrapper(reader_, (char *)&item, + sizeof(type))) { + return struct_pack::errc::no_buffer_space; + } + } + else { + return reader_.ignore(sizeof(type)) ? errc{} : errc::no_buffer_space; + } + } else if constexpr (unique_ptr) { bool has_value{}; if SP_UNLIKELY (!read_wrapper(reader_, @@ -646,7 +657,8 @@ class unpacker { code = detail::deserialize_varint(reader_, item); } else if constexpr (id == type_id::array_t) { - if constexpr (is_trivial_serializable::value) { + if constexpr (is_trivial_serializable::value && + is_little_endian_copyable) { if constexpr (NotSkip) { if SP_UNLIKELY (!read_wrapper( reader_, (char *)&item, @@ -782,16 +794,16 @@ class unpacker { view_reader_t, "The Reader isn't a view_reader, can't deserialize " "a string_view/span"); - static_assert( - sizeof(value_type) == 1 || is_system_little_endian, - "zero-copy in big endian is limit."); + static_assert(is_little_endian_copyable, + "zero-copy in big endian is limit."); const char *view = reader_.read_view(mem_sz); if SP_UNLIKELY (view == nullptr) { return struct_pack::errc::no_buffer_space; } item = {(value_type *)(view), (std::size_t)size64}; } - else { + else if constexpr (is_little_endian_copyable) { if SP_UNLIKELY (mem_sz >= PTRDIFF_MAX) unreachable(); else { @@ -802,6 +814,15 @@ class unpacker { } } } + else { + item.resize(size64); + for (auto &i : item) { + code = deserialize_one(i); + if SP_UNLIKELY (code != struct_pack::errc{}) { + return code; + } + } + } } else { return reader_.ignore(mem_sz) ? errc{} : errc::no_buffer_space; @@ -897,7 +918,7 @@ class unpacker { "struct_pack only support aggregated type, or you should " "add macro STRUCT_PACK_REFL(Type,field1,field2...)"); if constexpr (is_trivial_serializable::value && - is_system_little_endian) { + is_little_endian_copyable) { if constexpr (NotSkip) { if SP_UNLIKELY (!read_wrapper(reader_, (char *)&item)) { @@ -910,7 +931,7 @@ class unpacker { } } else if constexpr ((is_trivial_serializable::value && - !is_system_little_endian) || + !is_little_endian_copyable) || is_trivial_serializable::value) { visit_members(item, [&](auto &&...items) CONSTEXPR_INLINE_LAMBDA { int i = 1; diff --git a/src/struct_pack/tests/CMakeLists.txt b/src/struct_pack/tests/CMakeLists.txt index e0d13e98c..4f34d7a28 100644 --- a/src/struct_pack/tests/CMakeLists.txt +++ b/src/struct_pack/tests/CMakeLists.txt @@ -17,7 +17,13 @@ add_executable(struct_pack_test main.cpp ) add_test(NAME struct_pack_test COMMAND struct_pack_test) +include (TestBigEndian) +TEST_BIG_ENDIAN(IS_BIG_ENDIAN) +if(NOT IS_BIG_ENDIAN) +target_compile_definitions(struct_pack_test PRIVATE STRUCT_PACK_ENABLE_UNPORTABLE_TYPE TEST_IN_LITTLE_ENDIAN) +else() target_compile_definitions(struct_pack_test PRIVATE STRUCT_PACK_ENABLE_UNPORTABLE_TYPE) +endif() add_custom_command( TARGET struct_pack_test PRE_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory diff --git a/src/struct_pack/tests/test_data_struct2.cpp b/src/struct_pack/tests/test_data_struct2.cpp index b984cce9c..6ecbeb307 100644 --- a/src/struct_pack/tests/test_data_struct2.cpp +++ b/src/struct_pack/tests/test_data_struct2.cpp @@ -123,52 +123,33 @@ TEST_CASE("testing string_view deserialize") { CHECK(res); CHECK(res.value() == "hello"sv); } +#ifdef TEST_IN_LITTLE_ENDIAN { - std::u32string_view sv = U"你好"; - auto ret = serialize(sv); - auto res = deserialize(ret.data(), ret.size()); - CHECK(res); - CHECK(res.value() == sv); - } - { - std::u16string_view sv = u"你好"; - auto ret = serialize(sv); - auto res = deserialize(ret.data(), ret.size()); - CHECK(res); - CHECK(res.value() == sv); - } -#if __cpp_char8_t >= 201811L - { - std::u8string_view sv = u8"你好"; + auto sv = std::wstring_view(L"你好, struct pack"); auto ret = serialize(sv); - auto res = deserialize(ret.data(), ret.size()); - CHECK(res); - CHECK(res.value() == sv); - } -#endif - { - auto ret = serialize("hello"s); - auto res = deserialize(ret.data(), ret.size()); - CHECK(res); - CHECK(res.value() == "hello"sv); + std::wstring str; + auto ec = deserialize_to(str, ret.data(), ret.size()); + REQUIRE(ec == struct_pack::errc{}); + CHECK(str == sv); } { - std::u32string sv = U"你好"; + std::u32string_view sv = U"你好"; auto ret = serialize(sv); auto res = deserialize(ret.data(), ret.size()); CHECK(res); CHECK(res.value() == sv); } { - std::u16string sv = u"你好"; + std::u16string_view sv = u"你好"; auto ret = serialize(sv); auto res = deserialize(ret.data(), ret.size()); CHECK(res); CHECK(res.value() == sv); } +#endif #if __cpp_char8_t >= 201811L { - std::u8string sv = u8"你好"; + std::u8string_view sv = u8"你好"; auto ret = serialize(sv); auto res = deserialize(ret.data(), ret.size()); CHECK(res); @@ -215,51 +196,6 @@ TEST_CASE("test wide string") { } } -TEST_CASE("test string_view") { - using namespace std; - { - auto ret = serialize("hello struct pack"sv); - std::string str; - auto ec = deserialize_to(str, ret.data(), ret.size()); - REQUIRE(ec == struct_pack::errc{}); - CHECK(str == "hello struct pack"sv); - } - { - auto sv = std::wstring_view(L"你好, struct pack"); - auto ret = serialize(sv); - std::wstring str; - auto ec = deserialize_to(str, ret.data(), ret.size()); - REQUIRE(ec == struct_pack::errc{}); - CHECK(str == sv); - } -#if __cpp_char8_t >= 201811L - { - auto sv = std::u8string_view(u8"你好, struct pack"); - auto ret = serialize(sv); - std::u8string str; - auto ec = deserialize_to(str, ret.data(), ret.size()); - REQUIRE(ec == struct_pack::errc{}); - CHECK(str == sv); - } -#endif - { - auto sv = std::u16string_view(u"你好, struct pack"); - auto ret = serialize(sv); - std::u16string str; - auto ec = deserialize_to(str, ret.data(), ret.size()); - REQUIRE(ec == struct_pack::errc{}); - CHECK(str == sv); - } - { - auto sv = std::u32string_view(U"你好, struct pack"); - auto ret = serialize(sv); - std::u32string str; - auto ec = deserialize_to(str, ret.data(), ret.size()); - REQUIRE(ec == struct_pack::errc{}); - CHECK(str == sv); - } -} - TEST_CASE("char test") { { char ch = '1', ch2; @@ -570,7 +506,7 @@ bool test_equal(const trival_test::A_v3& v1, const trival_test::A_v3& v2) { (v1.b->b.b.b.a == v2.b->b.b.b.a && v1.b->b.b.b.c == v2.b->b.b.b.c && v1.b->b.b.b.d == v2.b->b.b.b.d)))); } - +#ifdef TEST_IN_LITTLE_ENDIAN TEST_CASE("testing trivial_view type") { trival_test::A_v1 a_v1 = { 'A', @@ -643,3 +579,4 @@ TEST_CASE("testing trivial_view type") { CHECK(test_equal(result.value(), a_v3)); } }; +#endif \ No newline at end of file diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index 9941c8f79..7dd49542e 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1205,8 +1205,13 @@ struct dspan_test6 { std::string hello; std::vector ar; }; +struct dspan_test7 { + std::string hello; + std::vector ar; +}; TEST_CASE("test dynamic span") { +#ifdef TEST_IN_LITTLE_ENDIAN SUBCASE("test dynamic span") { { std::vector ar = {1, 4, 5, 3}; @@ -1262,6 +1267,17 @@ TEST_CASE("test dynamic span") { CHECK(res.value().ar == ar); } } +#endif + SUBCASE("test dynamic span") { + { + std::vector ar = {1, 2, 132, 214}; + dspan_test7 s = {"Hello", ar}; + auto buffer = struct_pack::serialize(s); + auto res = struct_pack::deserialize(buffer); + CHECK(res.has_value()); + CHECK(res.value().hello == s.hello); + CHECK(res.value().ar == ar); + } + } } - #endif \ No newline at end of file From d85925f39d970622c394c1d8b14c8f41441783b5 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 15:29:56 +0800 Subject: [PATCH 21/32] fix --- .github/workflows/s390x.yml | 2 +- include/ylt/struct_pack/endian_wrapper.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index 7cea7b341..fb0bfa287 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -30,7 +30,7 @@ jobs: CXX=g++ CC=gcc cmake -B ${{github.workspace}}/build \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF -DBUILD_STRUCT_PB=OFF + -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF -DBUILD_STRUCT_PB=OFF -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF cmake --build ${{github.workspace}}/build -j cd ${{github.workspace}}/build/output/tests ./struct_pack_test \ No newline at end of file diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 7a5e87169..f679c592c 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -142,8 +142,8 @@ void write_wrapper(writer_t& writer, const char* data) { writer.write(&tmp1, block_size); #elif defined(__clang__) || defined(__GNUC__) auto tmp1 = __builtin_bswap64(*data), tmp2 = __builtin_bswap64(*data + 8); - writer.write((char*)&tmp2, block_size); - writer.write((char*)&tmp1, block_size); + writer.write((char*)&tmp2, 8); + writer.write((char*)&tmp1, 8); #else writer.write(data + 15, 1); writer.write(data + 14, 1); From 4849b89309f5cc3e8a8d13d93e058e76e6027c86 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 15:37:20 +0800 Subject: [PATCH 22/32] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index f679c592c..b8cfabd53 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -15,6 +15,7 @@ */ #pragma once #include +#include #include "reflection.hpp" #include "ylt/struct_pack/util.h" From fad4263690150dc12f6a0c7e1d315255a672a3f3 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 15:39:59 +0800 Subject: [PATCH 23/32] fix --- cmake/subdir.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/subdir.cmake b/cmake/subdir.cmake index 234a25fcb..527486465 100644 --- a/cmake/subdir.cmake +++ b/cmake/subdir.cmake @@ -27,13 +27,13 @@ foreach(child ${children}) get_filename_component(subdir_name ${child} NAME) string(TOUPPER ${subdir_name} subdir_name) if (BUILD_${subdir_name}) - if(BUILD_UNIT_TESTS AND EXISTS ${child}/examples) + if(BUILD_EXAMPLES AND EXISTS ${child}/examples) add_subdirectory(${child}/examples) endif() - if(BUILD_BENCHMARK AND EXISTS ${child}/tests) + if(BUILD_UNIT_TESTS AND EXISTS ${child}/tests) add_subdirectory(${child}/tests) endif() - if(BUILD_EXAMPLES AND EXISTS ${child}/benchmark) + if(BUILD_BENCHMARK AND EXISTS ${child}/benchmark) add_subdirectory(${child}/benchmark) endif() endif() From 614fac56154f3dda6200ee9d7f72d845dfadeaf7 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 16:11:42 +0800 Subject: [PATCH 24/32] fix format --- include/ylt/struct_pack/packer.hpp | 3 ++- include/ylt/struct_pack/reflection.hpp | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 6cdf7c52c..4de6324b3 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -247,7 +247,8 @@ class packer { } #endif if constexpr (trivially_copyable_container && - is_little_endian_copyable) { + is_little_endian_copyable) { using value_type = typename type::value_type; write_wrapper(writer_, (char *)item.data(), item.size()); diff --git a/include/ylt/struct_pack/reflection.hpp b/include/ylt/struct_pack/reflection.hpp index f5cb38c08..292efa77f 100644 --- a/include/ylt/struct_pack/reflection.hpp +++ b/include/ylt/struct_pack/reflection.hpp @@ -39,8 +39,6 @@ namespace struct_pack { namespace detail { - - template using get_args_type = remove_cvref_t>, From daad519b55838487b1afb8610eff1686296c9f3a Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 16:58:35 +0800 Subject: [PATCH 25/32] disable int128 in default --- include/ylt/struct_pack/reflection.hpp | 2 +- include/ylt/struct_pack/type_calculate.hpp | 2 +- include/ylt/struct_pack/type_id.hpp | 4 ++-- src/struct_pack/tests/test_serialize.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/ylt/struct_pack/reflection.hpp b/include/ylt/struct_pack/reflection.hpp index 292efa77f..e09f8973e 100644 --- a/include/ylt/struct_pack/reflection.hpp +++ b/include/ylt/struct_pack/reflection.hpp @@ -737,7 +737,7 @@ template return ignore_compatible_field; } else if constexpr (std::is_enum_v || std::is_fundamental_v || bitset -#if __GNUC__ || __clang__ +#if (__GNUC__ || __clang__) && defined(STRUCT_PACK_ENABLE_INT128) || std::is_same_v<__int128,T> || std::is_same_v #endif ) { diff --git a/include/ylt/struct_pack/type_calculate.hpp b/include/ylt/struct_pack/type_calculate.hpp index ef772f20b..19a9aba39 100644 --- a/include/ylt/struct_pack/type_calculate.hpp +++ b/include/ylt/struct_pack/type_calculate.hpp @@ -682,7 +682,7 @@ constexpr auto get_types() { string || container || optional || unique_ptr || is_variant_v || expected || array || c_array || std::is_same_v || bitset -#if __GNUC__ || __clang__ +#if (__GNUC__ || __clang__) && defined(STRUCT_PACK_ENABLE_INT128) || std::is_same_v<__int128, T> || std::is_same_v #endif diff --git a/include/ylt/struct_pack/type_id.hpp b/include/ylt/struct_pack/type_id.hpp index 18d3e598d..dba4aeccb 100644 --- a/include/ylt/struct_pack/type_id.hpp +++ b/include/ylt/struct_pack/type_id.hpp @@ -154,7 +154,7 @@ constexpr type_id get_integral_type() { "sizeof(bool)!=1, which is not supported."); return type_id::bool_t; } -#if __GNUC__ || __clang__ +#if (__GNUC__ || __clang__) && defined(STRUCT_PACK_ENABLE_INT128) //-std=gnu++20 else if constexpr (std::is_same_v<__int128, T>) { return type_id::int128_t; @@ -250,7 +250,7 @@ constexpr type_id get_type_id() { return get_integral_type>(); } else if constexpr (std::is_integral_v -#if __GNUC__ || __CLANG__ +#if (__GNUC__ || __clang__) && defined(STRUCT_PACK_ENABLE_INT128) || std::is_same_v<__int128, T> || std::is_same_v #endif diff --git a/src/struct_pack/tests/test_serialize.cpp b/src/struct_pack/tests/test_serialize.cpp index 7dd49542e..bd3852fc7 100644 --- a/src/struct_pack/tests/test_serialize.cpp +++ b/src/struct_pack/tests/test_serialize.cpp @@ -1019,7 +1019,7 @@ TEST_CASE("compatible convert to optional") { CHECK(b.value() == "hello world"); CHECK(a.value() == "hello world"); } -#if __GNUC__ || __clang__ +#if (__GNUC__ || __clang__) && defined(STRUCT_PACK_ENABLE_INT128) struct test_int_128 { __int128_t x; __uint128_t y; From 368a8a0a5b9dcda8b1a467f97ce00890c772f2b7 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Thu, 12 Oct 2023 18:05:34 +0800 Subject: [PATCH 26/32] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index b8cfabd53..fa5b42c14 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -94,10 +94,10 @@ void write_wrapper(writer_t& writer, const char* data) { } else if constexpr (block_size == 2) { #ifdef _MSC_VER - auto tmp = _byteswap_ushort(*data); + auto tmp = _byteswap_ushort(*(uint16_t*)data); writer.write(&tmp, block_size); #elif defined(__clang__) || defined(__GNUC__) - auto tmp = __builtin_bswap16(*data); + auto tmp = __builtin_bswap16(*(uint16_t*)data); writer.write((char*)&tmp, block_size); #else writer.write(data + 1, 1); @@ -106,10 +106,10 @@ void write_wrapper(writer_t& writer, const char* data) { } else if constexpr (block_size == 4) { #ifdef _MSC_VER - auto tmp = _byteswap_ulong(*data); + auto tmp = _byteswap_ulong(*(uint32_t*)data); writer.write(&tmp, block_size); #elif defined(__clang__) || defined(__GNUC__) - auto tmp = __builtin_bswap32(*data); + auto tmp = __builtin_bswap32(*(uint32_t*)data); writer.write((char*)&tmp, block_size); #else writer.write(data + 3, 1); @@ -120,10 +120,10 @@ void write_wrapper(writer_t& writer, const char* data) { } else if constexpr (block_size == 8) { #ifdef _MSC_VER - auto tmp = _byteswap_uint64(*data); + auto tmp = _byteswap_uint64(*(uint64_t*)data); writer.write(&tmp, block_size); #elif defined(__clang__) || defined(__GNUC__) - auto tmp = __builtin_bswap64(*data); + auto tmp = __builtin_bswap64(*(uint64_t*)data); writer.write((char*)&tmp, block_size); #else writer.write(data + 7, 1); @@ -138,7 +138,7 @@ void write_wrapper(writer_t& writer, const char* data) { } else if constexpr (block_size == 16) { #ifdef _MSC_VER - auto tmp1 = _byteswap_uint64(*data), tmp2 = _byteswap_uint64(*data + 8); + auto tmp1 = _byteswap_uint64(*(uint64_t*)data), tmp2 = _byteswap_uint64(*(uint64_t*)(data + 8)); writer.write(&tmp2, block_size); writer.write(&tmp1, block_size); #elif defined(__clang__) || defined(__GNUC__) @@ -190,7 +190,7 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { else { std::array tmp; bool res = static_cast(reader.read((char*)&tmp, block_size)); - if SP_UNLIKELY (res) { + if SP_UNLIKELY (!res) { return res; } if constexpr (block_size == 2) { From b4919aa08e9d46697c43af4ae87aeaf7ace2e27b Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Fri, 13 Oct 2023 11:23:46 +0800 Subject: [PATCH 27/32] fix endian --- include/ylt/struct_pack/endian_wrapper.hpp | 170 ++++--------- src/struct_pack/tests/test_compatible.cpp | 21 +- src/struct_pack/tests/test_varint.cpp | 272 ++++++++++++++------- 3 files changed, 255 insertions(+), 208 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index fa5b42c14..fedd6c258 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -14,8 +14,9 @@ * limitations under the License. */ #pragma once -#include #include +#include +#include #include "reflection.hpp" #include "ylt/struct_pack/util.h" @@ -87,82 +88,70 @@ template constexpr inline bool is_little_endian_copyable = is_system_little_endian || block_size == 1; +template +T swap_endian(T u) { + union { + T u; + unsigned char u8[sizeof(T)]; + } source, dest; + source.u = u; + for (size_t k = 0; k < sizeof(T); k++) + dest.u8[k] = source.u8[sizeof(T) - k - 1]; + return dest.u; +} + +inline uint16_t bswap16(uint16_t raw) { +#ifdef _MSC_VER + return _byteswap_ushort(raw); +#elif defined(__clang__) || defined(__GNUC__) + return __builtin_bswap16(raw); +#else + return swap_endian(raw); +#endif +}; + +inline uint32_t bswap32(uint32_t raw) { +#ifdef _MSC_VER + return _byteswap_ulong(raw); +#elif defined(__clang__) || defined(__GNUC__) + return __builtin_bswap32(raw); +#else + return swap_endian(raw); +#endif +}; + +inline uint64_t bswap64(uint64_t raw) { +#ifdef _MSC_VER + return _byteswap_uint64(raw); +#elif defined(__clang__) || defined(__GNUC__) + return __builtin_bswap64(raw); +#else + return swap_endian(raw); +#endif +}; + template void write_wrapper(writer_t& writer, const char* data) { if constexpr (is_system_little_endian || block_size == 1) { writer.write(data, block_size); } else if constexpr (block_size == 2) { -#ifdef _MSC_VER - auto tmp = _byteswap_ushort(*(uint16_t*)data); + auto tmp = bswap16(*(uint16_t*)data); writer.write(&tmp, block_size); -#elif defined(__clang__) || defined(__GNUC__) - auto tmp = __builtin_bswap16(*(uint16_t*)data); - writer.write((char*)&tmp, block_size); -#else - writer.write(data + 1, 1); - writer.write(data, 1); -#endif } else if constexpr (block_size == 4) { -#ifdef _MSC_VER - auto tmp = _byteswap_ulong(*(uint32_t*)data); + auto tmp = bswap32(*(uint32_t*)data); writer.write(&tmp, block_size); -#elif defined(__clang__) || defined(__GNUC__) - auto tmp = __builtin_bswap32(*(uint32_t*)data); - writer.write((char*)&tmp, block_size); -#else - writer.write(data + 3, 1); - writer.write(data + 2, 1); - writer.write(data + 1, 1); - writer.write(data, 1); -#endif } else if constexpr (block_size == 8) { -#ifdef _MSC_VER - auto tmp = _byteswap_uint64(*(uint64_t*)data); + auto tmp = bswap64(*(uint64_t*)data); writer.write(&tmp, block_size); -#elif defined(__clang__) || defined(__GNUC__) - auto tmp = __builtin_bswap64(*(uint64_t*)data); - writer.write((char*)&tmp, block_size); -#else - writer.write(data + 7, 1); - writer.write(data + 6, 1); - writer.write(data + 5, 1); - writer.write(data + 4, 1); - writer.write(data + 3, 1); - writer.write(data + 2, 1); - writer.write(data + 1, 1); - writer.write(data, 1); -#endif } else if constexpr (block_size == 16) { -#ifdef _MSC_VER - auto tmp1 = _byteswap_uint64(*(uint64_t*)data), tmp2 = _byteswap_uint64(*(uint64_t*)(data + 8)); + auto tmp1 = bswap64(*(uint64_t*)data), + tmp2 = bswap64(*(uint64_t*)(data + 8)); writer.write(&tmp2, block_size); writer.write(&tmp1, block_size); -#elif defined(__clang__) || defined(__GNUC__) - auto tmp1 = __builtin_bswap64(*data), tmp2 = __builtin_bswap64(*data + 8); - writer.write((char*)&tmp2, 8); - writer.write((char*)&tmp1, 8); -#else - writer.write(data + 15, 1); - writer.write(data + 14, 1); - writer.write(data + 13, 1); - writer.write(data + 12, 1); - writer.write(data + 11, 1); - writer.write(data + 10, 1); - writer.write(data + 9, 1); - writer.write(data + 8, 1); - writer.write(data + 7, 1); - writer.write(data + 6, 1); - writer.write(data + 5, 1); - writer.write(data + 4, 1); - writer.write(data + 3, 1); - writer.write(data + 2, 1); - writer.write(data + 1, 1); - writer.write(data, 1); -#endif } else { static_assert(!sizeof(writer), "illegal block size(should be 1,2,4,8,16)"); @@ -194,68 +183,17 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { return res; } if constexpr (block_size == 2) { -#ifdef _MSC_VER - *(uint16_t*)data = _byteswap_ushort(*(uint16_t*)&tmp); -#elif defined(__clang__) || defined(__GNUC__) - *(uint16_t*)data = __builtin_bswap16(*(uint16_t*)&tmp); -#else - data[0] = tmp[1]; - data[1] = tmp[0]; -#endif + *(uint16_t*)data = bswap16(*(uint16_t*)&tmp); } else if constexpr (block_size == 4) { -#ifdef _MSC_VER - *(uint32_t*)data = _byteswap_ulong(*(uint32_t*)&tmp); -#elif defined(__clang__) || defined(__GNUC__) - *(uint32_t*)data = __builtin_bswap32(*(uint32_t*)&tmp); -#else - data[0] = tmp[3]; - data[1] = tmp[2]; - data[2] = tmp[1]; - data[3] = tmp[0]; -#endif + *(uint32_t*)data = bswap32(*(uint32_t*)&tmp); } else if constexpr (block_size == 8) { -#ifdef _MSC_VER - *(uint64_t*)data = _byteswap_uint64(*(uint64_t*)&tmp); -#elif defined(__clang__) || defined(__GNUC__) - *(uint64_t*)data = __builtin_bswap64(*(uint64_t*)&tmp); -#else - data[0] = tmp[7]; - data[1] = tmp[6]; - data[2] = tmp[5]; - data[3] = tmp[4]; - data[4] = tmp[3]; - data[5] = tmp[2]; - data[6] = tmp[1]; - data[7] = tmp[0]; -#endif + *(uint64_t*)data = bswap64(*(uint64_t*)&tmp); } else if constexpr (block_size == 16) { -#ifdef _MSC_VER - *(uint64_t*)(data + 8) = _byteswap_uint64(*(uint64_t*)&tmp); - *(uint64_t*)data = _byteswap_uint64(*(uint64_t*)(&tmp + 8)); -#elif defined(__clang__) || defined(__GNUC__) - *(uint64_t*)(data + 8) = __builtin_bswap64(*(uint64_t*)&tmp); - *(uint64_t*)data = __builtin_bswap64(*(uint64_t*)(&tmp + 8)); -#else - data[0] = tmp[15]; - data[1] = tmp[14]; - data[2] = tmp[13]; - data[3] = tmp[12]; - data[4] = tmp[11]; - data[5] = tmp[10]; - data[6] = tmp[9]; - data[7] = tmp[8]; - data[8] = tmp[7]; - data[9] = tmp[6]; - data[10] = tmp[5]; - data[11] = tmp[4]; - data[12] = tmp[3]; - data[13] = tmp[2]; - data[14] = tmp[1]; - data[15] = tmp[0]; -#endif + *(uint64_t*)(data + 8) = bswap64(*(uint64_t*)&tmp); + *(uint64_t*)data = bswap64(*(uint64_t*)(&tmp + 8)); } else { static_assert(!sizeof(reader), diff --git a/src/struct_pack/tests/test_compatible.cpp b/src/struct_pack/tests/test_compatible.cpp index 5afd0f119..cb638bb24 100644 --- a/src/struct_pack/tests/test_compatible.cpp +++ b/src/struct_pack/tests/test_compatible.cpp @@ -3,6 +3,7 @@ #include "doctest.h" #include "test_struct.hpp" +#include "ylt/struct_pack/endian_wrapper.hpp" using namespace struct_pack; @@ -188,8 +189,11 @@ TEST_CASE("test compatible") { CHECK(sz == buffer.size()); CHECK(buffer[0] % 2 == 1); CHECK((buffer[4] & 0b11) == 1); - std::size_t r_sz = 0; + std::uint16_t r_sz = 0; memcpy(&r_sz, &buffer[5], 2); + if constexpr (!detail::is_system_little_endian) { + r_sz = detail::bswap16(r_sz); + } CHECK(r_sz == sz); auto big2 = deserialize>>>( @@ -211,8 +215,11 @@ TEST_CASE("test compatible") { CHECK(sz == buffer.size()); CHECK(buffer[0] % 2 == 1); CHECK((buffer[4] & 0b11) == 1); - std::size_t r_sz = 0; + std::uint16_t r_sz = 0; memcpy(&r_sz, &buffer[5], 2); + if constexpr (!detail::is_system_little_endian) { + r_sz = detail::bswap16(r_sz); + } CHECK(r_sz == sz); auto big2 = deserialize>>>( @@ -234,8 +241,11 @@ TEST_CASE("test compatible") { CHECK(sz == buffer.size()); CHECK(buffer[0] % 2 == 1); CHECK((buffer[4] & 0b11) == 1); - std::size_t r_sz = 0; + uint16_t r_sz = 0; memcpy(&r_sz, &buffer[5], 2); + if constexpr (!detail::is_system_little_endian) { + r_sz = detail::bswap16(r_sz); + } CHECK(r_sz == sz); auto big2 = deserialize>>(buffer); CHECK(big2); @@ -255,8 +265,11 @@ TEST_CASE("test compatible") { CHECK(sz == buffer.size()); CHECK(buffer[0] % 2 == 1); CHECK((buffer[4] & 0b11) == 2); - std::size_t r_sz = 0; + uint32_t r_sz = 0; memcpy(&r_sz, &buffer[5], 4); + if constexpr (!detail::is_system_little_endian) { + r_sz = detail::bswap32(r_sz); + } CHECK(r_sz == sz); auto big2 = deserialize>>(buffer); CHECK(big2); diff --git a/src/struct_pack/tests/test_varint.cpp b/src/struct_pack/tests/test_varint.cpp index 1f0a3637d..6916584c9 100644 --- a/src/struct_pack/tests/test_varint.cpp +++ b/src/struct_pack/tests/test_varint.cpp @@ -11,70 +11,80 @@ TEST_CASE("test uint32") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = 128 * 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = 128 * 128 * 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = 128 * 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint32_t v = UINT32_MAX; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } } @@ -84,133 +94,152 @@ TEST_CASE("test int32") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = 128 / 2 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = (128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = 128 / 2; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = (128 / 2 + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = 128 * 128 / 2 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = (128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = 128 * 128 / 2; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = (128 * 128 / 2 + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = 128 * 128 * 128 / 2 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = (128 * 128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = 128 * 128 * 128 / 2; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = (128 * 128 * 128 / 2 + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = 128 * 128 * 128 * 128 / 2 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = (128 * 128 * 128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = 128 * 128 * 128 * 128 / 2; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = (128 * 128 * 128 * 128 / 2 + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = INT32_MAX; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int32_t v = INT32_MIN; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } } @@ -220,119 +249,136 @@ TEST_CASE("test uint64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 128 * 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 128 * 128 * 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 128 * 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 6); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 6); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 7); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128 * 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 7); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 8); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128 * 128 * 128 * 128 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 8); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128 * 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 9); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = @@ -340,21 +386,24 @@ TEST_CASE("test uint64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 9); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = 1ull * 128 * 128 * 128 * 128 * 128 * 128 * 128 * 128 * 128; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 10); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_uint64_t v = UINT64_MAX; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 10); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } } @@ -364,196 +413,224 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = 128 / 2 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 1); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = 128 / 2; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (128 / 2 + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = 128 * 128 / 2 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 2); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = 128 * 128 / 2; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (128 * 128 / 2 + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = 128 * 128 * 128 / 2 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (128 * 128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 3); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = 128 * 128 * 128 / 2; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (128 * 128 * 128 / 2 + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = 128 * 128 * 128 * 128 / 2 - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (128 * 128 * 128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 4); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = 128 * 128 * 128 * 128 / 2; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (128 * 128 * 128 * 128 / 2 + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 / 2) - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 5); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 / 2); auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 6); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = ((1ll * 128 * 128 * 128 * 128 * 128 / 2) + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 6); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 * 128 / 2) - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 6); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 6); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 * 128 / 2); auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 7); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = ((1ll * 128 * 128 * 128 * 128 * 128 * 128 / 2) + 1) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 7); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 * 128 * 128 / 2) - 1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 7); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 * 128 * 128 / 2) * -1; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 7); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 * 128 * 128 / 2); auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 8); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = @@ -561,7 +638,8 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 8); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = @@ -569,7 +647,8 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 8); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = @@ -577,14 +656,16 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 8); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = (1ll * 128 * 128 * 128 * 128 * 128 * 128 * 128 * 128 / 2); auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 9); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = @@ -592,7 +673,8 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 9); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = @@ -600,7 +682,8 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 9); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = @@ -608,7 +691,8 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 9); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = @@ -616,7 +700,8 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 10); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = @@ -625,21 +710,24 @@ TEST_CASE("test int64") { auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 10); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = INT64_MAX; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 10); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } { var_int64_t v = INT64_MIN; auto buf = serialize(v); auto v2 = deserialize(buf); CHECK(detail::calculate_payload_size(v).total == 10); - CHECK(v == v2); + CHECK(v2.has_value()); + CHECK(v == v2.value()); } } @@ -667,44 +755,52 @@ TEST_CASE("test nested") { VarRect rect{INT32_MAX, INT32_MIN, 123, 0}; auto buf = serialize(rect); auto rect2 = deserialize>(buf); + CHECK(rect2.has_value()); CHECK(rect == rect2); nested_object2 obj{123, "jfslkf", person{24, "Hello"}, complicated_object{}, rect}; auto buf2 = serialize(obj); auto obj2 = deserialize>(buf2); + CHECK(obj2.has_value()); CHECK(obj2 == obj); } { VarRect rect{INT64_MAX, INT64_MIN, 123, 0}; auto buf = serialize(rect); auto rect2 = deserialize>(buf); + CHECK(rect2.has_value()); CHECK(rect == rect2); nested_object2 obj{123, "jfslkf", person{24, "Hello"}, complicated_object{}, rect}; auto buf2 = serialize(obj); auto obj2 = deserialize>(buf2); + CHECK(obj2.has_value()); CHECK(obj2 == obj); } { VarRect rect{UINT32_MAX, 21321343, 123, 0}; auto buf = serialize(rect); auto rect2 = deserialize>(buf); + CHECK(rect2.has_value()); CHECK(rect == rect2); nested_object2 obj{123, "jfslkf", person{24, "Hello"}, complicated_object{}, rect}; auto buf2 = serialize(obj); auto obj2 = deserialize>(buf2); + CHECK(obj2.has_value()); CHECK(obj2 == obj); } { VarRect rect{UINT64_MAX, 1233143, 123, 0}; auto buf = serialize(rect); auto rect2 = deserialize>(buf); + CHECK(rect2.has_value()); CHECK(rect == rect2); nested_object2 obj{123, "jfslkf", person{24, "Hello"}, complicated_object{}, rect}; auto buf2 = serialize(obj); auto obj2 = deserialize>(buf2); + CHECK(obj2.has_value()); CHECK(obj2 == obj); } } From 5ce0e8fda00de821baefed8d1d56e62993c903f6 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Fri, 13 Oct 2023 11:44:49 +0800 Subject: [PATCH 28/32] fix --- .github/workflows/s390x.yml | 2 +- include/ylt/struct_pack/endian_wrapper.hpp | 10 +++++----- src/struct_pack/tests/test_compatible.cpp | 4 ---- src/struct_pack/tests/test_compile_time_calculate.cpp | 3 +++ 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/s390x.yml b/.github/workflows/s390x.yml index fb0bfa287..a5899873a 100644 --- a/.github/workflows/s390x.yml +++ b/.github/workflows/s390x.yml @@ -29,7 +29,7 @@ jobs: lscpu | grep Endian CXX=g++ CC=gcc cmake -B ${{github.workspace}}/build \ - -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_BUILD_TYPE=Debug \ -DBUILD_CORO_HTTP=OFF -DBUILD_CORO_IO=OFF -DBUILD_CORO_RPC=OFF -DBUILD_EASYLOG=OFF -DBUILD_STRUCT_JSON=OFF -DBUILD_STRUCT_XML=OFF -DBUILD_STRUCT_YAML=OFF -DBUILD_UTIL=OFF -DBUILD_STRUCT_PB=OFF -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF cmake --build ${{github.workspace}}/build -j cd ${{github.workspace}}/build/output/tests diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index fedd6c258..6537b7beb 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -137,21 +137,21 @@ void write_wrapper(writer_t& writer, const char* data) { } else if constexpr (block_size == 2) { auto tmp = bswap16(*(uint16_t*)data); - writer.write(&tmp, block_size); + writer.write((char*)&tmp, block_size); } else if constexpr (block_size == 4) { auto tmp = bswap32(*(uint32_t*)data); - writer.write(&tmp, block_size); + writer.write((char*)&tmp, block_size); } else if constexpr (block_size == 8) { auto tmp = bswap64(*(uint64_t*)data); - writer.write(&tmp, block_size); + writer.write((char*)&tmp, block_size); } else if constexpr (block_size == 16) { auto tmp1 = bswap64(*(uint64_t*)data), tmp2 = bswap64(*(uint64_t*)(data + 8)); - writer.write(&tmp2, block_size); - writer.write(&tmp1, block_size); + writer.write((char*)&tmp2, block_size); + writer.write((char*)&tmp1, block_size); } else { static_assert(!sizeof(writer), "illegal block size(should be 1,2,4,8,16)"); diff --git a/src/struct_pack/tests/test_compatible.cpp b/src/struct_pack/tests/test_compatible.cpp index cb638bb24..0e74abeb6 100644 --- a/src/struct_pack/tests/test_compatible.cpp +++ b/src/struct_pack/tests/test_compatible.cpp @@ -187,7 +187,6 @@ TEST_CASE("test compatible") { CHECK(sz == 255); auto buffer = serialize(big); CHECK(sz == buffer.size()); - CHECK(buffer[0] % 2 == 1); CHECK((buffer[4] & 0b11) == 1); std::uint16_t r_sz = 0; memcpy(&r_sz, &buffer[5], 2); @@ -213,7 +212,6 @@ TEST_CASE("test compatible") { CHECK(sz == 256); auto buffer = serialize(big); CHECK(sz == buffer.size()); - CHECK(buffer[0] % 2 == 1); CHECK((buffer[4] & 0b11) == 1); std::uint16_t r_sz = 0; memcpy(&r_sz, &buffer[5], 2); @@ -239,7 +237,6 @@ TEST_CASE("test compatible") { CHECK(sz == 65535); auto buffer = serialize(big); CHECK(sz == buffer.size()); - CHECK(buffer[0] % 2 == 1); CHECK((buffer[4] & 0b11) == 1); uint16_t r_sz = 0; memcpy(&r_sz, &buffer[5], 2); @@ -263,7 +260,6 @@ TEST_CASE("test compatible") { CHECK(sz == 65538); auto buffer = serialize(big); CHECK(sz == buffer.size()); - CHECK(buffer[0] % 2 == 1); CHECK((buffer[4] & 0b11) == 2); uint32_t r_sz = 0; memcpy(&r_sz, &buffer[5], 4); diff --git a/src/struct_pack/tests/test_compile_time_calculate.cpp b/src/struct_pack/tests/test_compile_time_calculate.cpp index 268a808fe..e5a06b02c 100644 --- a/src/struct_pack/tests/test_compile_time_calculate.cpp +++ b/src/struct_pack/tests/test_compile_time_calculate.cpp @@ -63,6 +63,9 @@ TEST_CASE("test hash conflict detected") { int32_t value = 42; auto ret = serialize(value); auto fake_hash = struct_pack::get_type_code() | 0b1; + if constexpr (!detail::is_system_little_endian) { + fake_hash = detail::bswap32(fake_hash); + } memcpy(ret.data(), &fake_hash, sizeof(fake_hash)); auto res = deserialize(ret); CHECK(!res); From eddf8e789b2bd211bd212c8f827f4718228f513a Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Fri, 13 Oct 2023 13:45:52 +0800 Subject: [PATCH 29/32] fix --- include/ylt/struct_pack/varint.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/ylt/struct_pack/varint.hpp b/include/ylt/struct_pack/varint.hpp index d5689cc60..8a03df84c 100644 --- a/include/ylt/struct_pack/varint.hpp +++ b/include/ylt/struct_pack/varint.hpp @@ -212,7 +212,13 @@ STRUCT_PACK_INLINE void serialize_varint(writer& writer_, const T& t) { write_wrapper(writer_, (char*)&temp); v >>= 7; } - write_wrapper(writer_, (char*)&v); + if constexpr (is_system_little_endian) { + write_wrapper(writer_, (char*)&v); + } + else { + uint8_t tmp = v; + write_wrapper(writer_, (char*)&tmp); + } } #if __cpp_concepts >= 201907L template From 5bca84b0a194863ab486b4e9ce30b452cb781930 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Fri, 13 Oct 2023 14:16:06 +0800 Subject: [PATCH 30/32] [no ci] add doc --- website/docs/en/struct_pack/struct_pack_tips.md | 14 +++++++++++--- website/docs/zh/struct_pack/struct_pack_tips.md | 11 +++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/website/docs/en/struct_pack/struct_pack_tips.md b/website/docs/en/struct_pack/struct_pack_tips.md index ddde4e570..612fccb5e 100644 --- a/website/docs/en/struct_pack/struct_pack_tips.md +++ b/website/docs/en/struct_pack/struct_pack_tips.md @@ -1,21 +1,29 @@ # struct_pack Tips This document introcude some tips about struct_pack -## compiler requires +## requirements -struct_pack need compiler at least support C++17. +struct_pack need compiler at least support C++17. We have test it on GCC/Clang/MSVC. | compiler | tested min version | | ----------- | ------------------ | | GCC | 9.1 | -| Clang | 6.0 | +| Clang/Apple Clang | 6.0 | | MSVC | Visual Studio 2019 (MSVC19.20) | +struct_pack is cross-platform. struct_pack can work in both big-endian and little-endian. struct_pack support 32bit arch or 64bit arch. + When C++20 standard was enabled, struct_pack has better performance by support memcpy user-defined trivial continuous container. In C++17, struct_pack only support memcpy container `std::array`, `std::vector`, `std::basic_string` and `std::basic_string_view`. + +## endian + +struct_pack will save the data as little-endian even if the arch is big-endian. The trivial-copy optimize and zero-copy optimize will be disable when the object is width than 1 byte. so struct_pack performance in little-endian is better than big-endian. + ## macro | macro | description | | ----------- | ------------------ | | STRUCT_PACK_OPTIMIZE | Allow more extremed loop unrolling to get better performance, but it cost more compile time. | | STRUCT_PACK_ENABLE_UNPORTABLE_TYPE | Enable serialize unportable type, such as wchar_t and wstring. Deserialize them in other platform is ubdefined bevaior. | +| STRUCT_PACK_ENABLE_INT128 | Enable serialize __int128 and __uint128. Not all compiler support it. ## How to speed up serialization/deserialization 1. use string_view instead of string, use span instead of vector/array. 2. move trivial field to a standlone struct, so that struct_pack could copy it faster. diff --git a/website/docs/zh/struct_pack/struct_pack_tips.md b/website/docs/zh/struct_pack/struct_pack_tips.md index 05c92cd4f..b73bcf6ef 100644 --- a/website/docs/zh/struct_pack/struct_pack_tips.md +++ b/website/docs/zh/struct_pack/struct_pack_tips.md @@ -1,8 +1,8 @@ # struct_pack 使用提示 本文档介绍一些struct_pack的细节,包括一些优化和约束。 -## 编译器支持 +## 约束条件 -struct_pack需要你的编译器良好的支持C++17标准。 +struct_pack需要你的编译器良好的支持C++17标准。我们已在GCC/Clang/MSVC编译器下进行了测试 | 编译器 | 已测试的最低版本 | | ----------- | ------------------ | @@ -10,14 +10,21 @@ struct_pack需要你的编译器良好的支持C++17标准。 | Clang | 6.0 | | MSVC | Visual Studio 2019 (MSVC19.20) | +struct_pack是跨平台的。它可以在小端/大端,32位/64位架构下正常工作。 + 当启用C++20标准时,struct_pack具有更好的性能,能支持自定义的内存连续的可平凡拷贝容器。 而在C++17标准下,struct_pack仅支持标准库中已有的`std::array`, `std::vector`, `std::basic_string` 和 `std::basic_string_view`,不支持其他自定义容器类型的平凡拷贝。 +## 端序 + +struct_pack即使在大端架构下,也会将数据保存为小端格式。当对象的宽度大于1字节时,平凡复制优化和零复制优化将被禁用。因此,struct_pack在小端架构中的性能优于大端架构。 + ## 宏 | 宏 | 作用 | | ----------- | ------------------ | | STRUCT_PACK_OPTIMIZE |增加模板实例化的数量,通过牺牲编译时间和二进制体积来换取更好的性能 | |STRUCT_PACK_ENABLE_UNPORTABLE_TYPE |允许序列化unportable的类型,如wchar_t和wstring,请注意,这些类型序列化出的二进制数据可能无法正常的在其他平台下反序列化| +| STRUCT_PACK_ENABLE_INT128 | 允许序列化128位整数,包括__uint128和__int128类型。请注意,只有部分编译器在部分架构下支持该类型。 ## 如何让序列化or反序列化更加高效 1. 考虑使用string_view代替string, 使用span代替vector/array; 2. 把平凡字段封装到一个单独的结构体中,优化拷贝速度; From ba38195330ee1f87de0ca29afab0a6dd6254e5ec Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Fri, 13 Oct 2023 15:13:20 +0800 Subject: [PATCH 31/32] resolve conversation --- include/ylt/struct_pack/endian_wrapper.hpp | 25 ++++++++++------------ include/ylt/struct_pack/packer.hpp | 14 ++++++------ include/ylt/struct_pack/unpacker.hpp | 19 ++++++++-------- 3 files changed, 26 insertions(+), 32 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 6537b7beb..02827bae8 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -157,15 +157,13 @@ void write_wrapper(writer_t& writer, const char* data) { static_assert(!sizeof(writer), "illegal block size(should be 1,2,4,8,16)"); } } -template -void write_wrapper(writer_t& writer, const char* data, - std::size_t block_count) { - if constexpr (is_system_little_endian || block_size == 1) { - auto total_sz = block_count * block_size; - if SP_UNLIKELY (total_sz >= PTRDIFF_MAX) +template +void write_bytes_array(writer_t& writer, const char* data, std::size_t length) { + if constexpr (is_system_little_endian) { + if SP_UNLIKELY (length >= PTRDIFF_MAX) unreachable(); else - writer.write(data, total_sz); + writer.write(data, length); } else { static_assert(!sizeof(writer_t), "illegal use"); @@ -202,15 +200,14 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { return true; } } -template -bool read_wrapper(reader_t& reader, char* SP_RESTRICT data, - std::size_t block_count) { - if constexpr (is_system_little_endian || block_size == 1) { - auto total_sz = block_count * block_size; - if SP_UNLIKELY (total_sz >= PTRDIFF_MAX) +template +bool read_bytes_array(reader_t& reader, char* SP_RESTRICT data, + std::size_t length) { + if constexpr (is_system_little_endian) { + if SP_UNLIKELY (length >= PTRDIFF_MAX) unreachable(); else - return static_cast(reader.read(data, total_sz)); + return static_cast(reader.read(data, length)); } else { static_assert(!sizeof(reader_t), "illegal use"); diff --git a/include/ylt/struct_pack/packer.hpp b/include/ylt/struct_pack/packer.hpp index 4de6324b3..d9310bd2d 100644 --- a/include/ylt/struct_pack/packer.hpp +++ b/include/ylt/struct_pack/packer.hpp @@ -19,8 +19,6 @@ #include "calculate_size.hpp" #include "endian_wrapper.hpp" #include "reflection.hpp" -#include "ylt/struct_pack/type_id.hpp" -#include "ylt/struct_pack/util.h" namespace struct_pack::detail { template < #if __cpp_concepts >= 201907L @@ -121,7 +119,8 @@ class packer { if constexpr (check_if_add_type_literal()) { constexpr auto type_literal = struct_pack::get_type_literal(); - write_wrapper<1>(writer_, type_literal.data(), type_literal.size() + 1); + write_bytes_array(writer_, type_literal.data(), + type_literal.size() + 1); } } } @@ -163,7 +162,7 @@ class packer { write_wrapper(writer_, (char *)&item); } else if constexpr (id == type_id::bitset_t) { - write_wrapper(writer_, (char *)&item, sizeof(item)); + write_bytes_array(writer_, (char *)&item, sizeof(item)); } else if constexpr (unique_ptr) { bool has_value = (item != nullptr); @@ -193,8 +192,7 @@ class packer { else if constexpr (id == type_id::array_t) { if constexpr (is_trivial_serializable::value && is_little_endian_copyable) { - write_wrapper(writer_, (char *)&item, - sizeof(type) / sizeof(item[0])); + write_bytes_array(writer_, (char *)&item, sizeof(type)); } else { for (const auto &i : item) { @@ -250,8 +248,8 @@ class packer { is_little_endian_copyable) { using value_type = typename type::value_type; - write_wrapper(writer_, (char *)item.data(), - item.size()); + write_bytes_array(writer_, (char *)item.data(), + item.size() * sizeof(typename type::value_type)); return; } else { diff --git a/include/ylt/struct_pack/unpacker.hpp b/include/ylt/struct_pack/unpacker.hpp index 859c5fc3d..65f3a592e 100644 --- a/include/ylt/struct_pack/unpacker.hpp +++ b/include/ylt/struct_pack/unpacker.hpp @@ -44,7 +44,6 @@ #include "type_id.hpp" #include "type_trait.hpp" #include "varint.hpp" -#include "ylt/struct_pack/util.h" namespace struct_pack { @@ -488,7 +487,7 @@ class unpacker { } else { char buffer[literal.size() + 1]; - if SP_UNLIKELY (!read_wrapper<1>(reader_, buffer, literal.size() + 1)) { + if SP_UNLIKELY (!read_bytes_array(reader_, buffer, literal.size() + 1)) { return errc::no_buffer_space; } if SP_UNLIKELY (memcmp(buffer, literal.data(), literal.size() + 1)) { @@ -613,8 +612,8 @@ class unpacker { } else if constexpr (id == type_id::bitset_t) { if constexpr (NotSkip) { - if SP_UNLIKELY (!read_wrapper(reader_, (char *)&item, - sizeof(type))) { + if SP_UNLIKELY (!read_bytes_array(reader_, (char *)&item, + sizeof(type))) { return struct_pack::errc::no_buffer_space; } } @@ -660,9 +659,8 @@ class unpacker { if constexpr (is_trivial_serializable::value && is_little_endian_copyable) { if constexpr (NotSkip) { - if SP_UNLIKELY (!read_wrapper( - reader_, (char *)&item, - sizeof(item) / sizeof(item[0]))) { + if SP_UNLIKELY (!read_bytes_array(reader_, (char *)&item, + sizeof(item))) { return struct_pack::errc::no_buffer_space; } } @@ -681,12 +679,12 @@ class unpacker { } } else if constexpr (container) { - uint8_t size8; uint16_t size16; uint32_t size32; uint64_t size64; bool result; if constexpr (size_type == 1) { + uint8_t size8; if SP_UNLIKELY (!read_wrapper(reader_, (char *)&size8)) { return struct_pack::errc::no_buffer_space; } @@ -808,8 +806,9 @@ class unpacker { unreachable(); else { item.resize(size64); - if SP_UNLIKELY (!read_wrapper( - reader_, (char *)item.data(), size64)) { + if SP_UNLIKELY (!read_bytes_array( + reader_, (char *)item.data(), + size64 * sizeof(value_type))) { return struct_pack::errc::no_buffer_space; } } From dad91765203fc75a80a89770391009620fe5acc8 Mon Sep 17 00:00:00 2001 From: "Zezheng.Li" Date: Fri, 13 Oct 2023 15:21:26 +0800 Subject: [PATCH 32/32] fix --- include/ylt/struct_pack/endian_wrapper.hpp | 26 +++++++--------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/include/ylt/struct_pack/endian_wrapper.hpp b/include/ylt/struct_pack/endian_wrapper.hpp index 02827bae8..7b23fa07c 100644 --- a/include/ylt/struct_pack/endian_wrapper.hpp +++ b/include/ylt/struct_pack/endian_wrapper.hpp @@ -159,15 +159,10 @@ void write_wrapper(writer_t& writer, const char* data) { } template void write_bytes_array(writer_t& writer, const char* data, std::size_t length) { - if constexpr (is_system_little_endian) { - if SP_UNLIKELY (length >= PTRDIFF_MAX) - unreachable(); - else - writer.write(data, length); - } - else { - static_assert(!sizeof(writer_t), "illegal use"); - } + if SP_UNLIKELY (length >= PTRDIFF_MAX) + unreachable(); + else + writer.write(data, length); } template bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { @@ -203,14 +198,9 @@ bool read_wrapper(reader_t& reader, char* SP_RESTRICT data) { template bool read_bytes_array(reader_t& reader, char* SP_RESTRICT data, std::size_t length) { - if constexpr (is_system_little_endian) { - if SP_UNLIKELY (length >= PTRDIFF_MAX) - unreachable(); - else - return static_cast(reader.read(data, length)); - } - else { - static_assert(!sizeof(reader_t), "illegal use"); - } + if SP_UNLIKELY (length >= PTRDIFF_MAX) + unreachable(); + else + return static_cast(reader.read(data, length)); } }; // namespace struct_pack::detail \ No newline at end of file