From e40224d0c77674348bf0a518365208bc118f39a4 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 17 May 2021 13:21:23 +0200 Subject: [PATCH 1/6] Remove unused float serialization --- src/Makefile.am | 1 + src/serialize.h | 19 +++--------------- src/test/fuzz/float.cpp | 12 ----------- src/test/fuzz/util.h | 2 -- src/test/serialize_tests.cpp | 39 ------------------------------------ src/util/types.h | 11 ++++++++++ 6 files changed, 15 insertions(+), 69 deletions(-) create mode 100644 src/util/types.h diff --git a/src/Makefile.am b/src/Makefile.am index 770bb76226a..c5a7d0ad317 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -264,6 +264,7 @@ BITCOIN_CORE_H = \ util/tokenpipe.h \ util/trace.h \ util/translation.h \ + util/types.h \ util/ui_change_type.h \ util/url.h \ util/vector.h \ diff --git a/src/serialize.h b/src/serialize.h index d9ca984f9cd..276f643c7f1 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -23,6 +23,7 @@ #include #include +#include /** * The maximum size of a serialized object in bytes or number of elements @@ -129,13 +130,6 @@ inline uint64_t ser_double_to_uint64(double x) static_assert(sizeof(tmp) == sizeof(x), "double and uint64_t assumed to have the same size"); return tmp; } -inline uint32_t ser_float_to_uint32(float x) -{ - uint32_t tmp; - std::memcpy(&tmp, &x, sizeof(x)); - static_assert(sizeof(tmp) == sizeof(x), "float and uint32_t assumed to have the same size"); - return tmp; -} inline double ser_uint64_to_double(uint64_t y) { double tmp; @@ -143,13 +137,6 @@ inline double ser_uint64_to_double(uint64_t y) static_assert(sizeof(tmp) == sizeof(y), "double and uint64_t assumed to have the same size"); return tmp; } -inline float ser_uint32_to_float(uint32_t y) -{ - float tmp; - std::memcpy(&tmp, &y, sizeof(y)); - static_assert(sizeof(tmp) == sizeof(y), "float and uint32_t assumed to have the same size"); - return tmp; -} ///////////////////////////////////////////////////////////////// @@ -234,7 +221,7 @@ template inline void Serialize(Stream& s, int32_t a ) { ser_wri template inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); } template inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); } template inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } -template inline void Serialize(Stream& s, float a ) { ser_writedata32(s, ser_float_to_uint32(a)); } +template inline void Serialize(Stream& s, float a ) { static_assert(ALWAYS_FALSE, "Not implemented"); } template inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); } template inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); } template inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); } @@ -252,7 +239,7 @@ template inline void Unserialize(Stream& s, int32_t& a ) { a = template inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); } template inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); } template inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } -template inline void Unserialize(Stream& s, float& a ) { a = ser_uint32_to_float(ser_readdata32(s)); } +template inline void Unserialize(Stream& s, float& a ) { static_assert(ALWAYS_FALSE, "Not implemented"); } template inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); } template inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); } template inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); } diff --git a/src/test/fuzz/float.cpp b/src/test/fuzz/float.cpp index d18a87d1774..f806c82d14b 100644 --- a/src/test/fuzz/float.cpp +++ b/src/test/fuzz/float.cpp @@ -27,16 +27,4 @@ FUZZ_TARGET(float) stream >> d_deserialized; assert(d == d_deserialized); } - - { - const float f = fuzzed_data_provider.ConsumeFloatingPoint(); - (void)memusage::DynamicUsage(f); - assert(ser_uint32_to_float(ser_float_to_uint32(f)) == f); - - CDataStream stream(SER_NETWORK, INIT_PROTO_VERSION); - stream << f; - float f_deserialized; - stream >> f_deserialized; - assert(f == f_deserialized); - } } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 48b78778963..ac8df782be8 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -513,7 +513,6 @@ void WriteToStream(FuzzedDataProvider& fuzzed_data_provider, Stream& stream) noe WRITE_TO_STREAM_CASE(uint32_t, fuzzed_data_provider.ConsumeIntegral()), WRITE_TO_STREAM_CASE(int64_t, fuzzed_data_provider.ConsumeIntegral()), WRITE_TO_STREAM_CASE(uint64_t, fuzzed_data_provider.ConsumeIntegral()), - WRITE_TO_STREAM_CASE(float, fuzzed_data_provider.ConsumeFloatingPoint()), WRITE_TO_STREAM_CASE(double, fuzzed_data_provider.ConsumeFloatingPoint()), WRITE_TO_STREAM_CASE(std::string, fuzzed_data_provider.ConsumeRandomLengthString(32)), WRITE_TO_STREAM_CASE(std::vector, ConsumeRandomLengthIntegralVector(fuzzed_data_provider))); @@ -545,7 +544,6 @@ void ReadFromStream(FuzzedDataProvider& fuzzed_data_provider, Stream& stream) no READ_FROM_STREAM_CASE(uint32_t), READ_FROM_STREAM_CASE(int64_t), READ_FROM_STREAM_CASE(uint64_t), - READ_FROM_STREAM_CASE(float), READ_FROM_STREAM_CASE(double), READ_FROM_STREAM_CASE(std::string), READ_FROM_STREAM_CASE(std::vector)); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index f77cda7ba2e..0fdf4818236 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -70,7 +70,6 @@ BOOST_AUTO_TEST_CASE(sizes) BOOST_CHECK_EQUAL(sizeof(uint32_t), GetSerializeSize(uint32_t(0), 0)); BOOST_CHECK_EQUAL(sizeof(int64_t), GetSerializeSize(int64_t(0), 0)); BOOST_CHECK_EQUAL(sizeof(uint64_t), GetSerializeSize(uint64_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(float), GetSerializeSize(float(0), 0)); BOOST_CHECK_EQUAL(sizeof(double), GetSerializeSize(double(0), 0)); // Bool is serialized as char BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(bool(0), 0)); @@ -85,30 +84,10 @@ BOOST_AUTO_TEST_CASE(sizes) BOOST_CHECK_EQUAL(GetSerializeSize(uint32_t(0), 0), 4U); BOOST_CHECK_EQUAL(GetSerializeSize(int64_t(0), 0), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(uint64_t(0), 0), 8U); - BOOST_CHECK_EQUAL(GetSerializeSize(float(0), 0), 4U); BOOST_CHECK_EQUAL(GetSerializeSize(double(0), 0), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(bool(0), 0), 1U); } -BOOST_AUTO_TEST_CASE(floats_conversion) -{ - // Choose values that map unambiguously to binary floating point to avoid - // rounding issues at the compiler side. - BOOST_CHECK_EQUAL(ser_uint32_to_float(0x00000000), 0.0F); - BOOST_CHECK_EQUAL(ser_uint32_to_float(0x3f000000), 0.5F); - BOOST_CHECK_EQUAL(ser_uint32_to_float(0x3f800000), 1.0F); - BOOST_CHECK_EQUAL(ser_uint32_to_float(0x40000000), 2.0F); - BOOST_CHECK_EQUAL(ser_uint32_to_float(0x40800000), 4.0F); - BOOST_CHECK_EQUAL(ser_uint32_to_float(0x44444444), 785.066650390625F); - - BOOST_CHECK_EQUAL(ser_float_to_uint32(0.0F), 0x00000000U); - BOOST_CHECK_EQUAL(ser_float_to_uint32(0.5F), 0x3f000000U); - BOOST_CHECK_EQUAL(ser_float_to_uint32(1.0F), 0x3f800000U); - BOOST_CHECK_EQUAL(ser_float_to_uint32(2.0F), 0x40000000U); - BOOST_CHECK_EQUAL(ser_float_to_uint32(4.0F), 0x40800000U); - BOOST_CHECK_EQUAL(ser_float_to_uint32(785.066650390625F), 0x44444444U); -} - BOOST_AUTO_TEST_CASE(doubles_conversion) { // Choose values that map unambiguously to binary floating point to avoid @@ -135,26 +114,8 @@ Python code to generate the below hashes: def dsha256(x): return hashlib.sha256(hashlib.sha256(x).digest()).digest() - reversed_hex(dsha256(''.join(struct.pack('> j; - BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); - } -} - BOOST_AUTO_TEST_CASE(doubles) { CDataStream ss(SER_DISK, 0); diff --git a/src/util/types.h b/src/util/types.h new file mode 100644 index 00000000000..0047b00026b --- /dev/null +++ b/src/util/types.h @@ -0,0 +1,11 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_TYPES_H +#define BITCOIN_UTIL_TYPES_H + +template +inline constexpr bool ALWAYS_FALSE{false}; + +#endif // BITCOIN_UTIL_TYPES_H From 2be4cd94f4c7d92a4287971233a20d68db81c9c9 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 16 May 2021 17:23:17 -0700 Subject: [PATCH 2/6] Add platform-independent float encoder/decoder --- src/Makefile.am | 2 ++ src/util/serfloat.cpp | 64 +++++++++++++++++++++++++++++++++++++++++++ src/util/serfloat.h | 16 +++++++++++ 3 files changed, 82 insertions(+) create mode 100644 src/util/serfloat.cpp create mode 100644 src/util/serfloat.h diff --git a/src/Makefile.am b/src/Makefile.am index c5a7d0ad317..f3f58b41b7e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -253,6 +253,7 @@ BITCOIN_CORE_H = \ util/moneystr.h \ util/rbf.h \ util/readwritefile.h \ + util/serfloat.h \ util/settings.h \ util/sock.h \ util/spanparsing.h \ @@ -595,6 +596,7 @@ libbitcoin_util_a_SOURCES = \ util/settings.cpp \ util/thread.cpp \ util/threadnames.cpp \ + util/serfloat.cpp \ util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ diff --git a/src/util/serfloat.cpp b/src/util/serfloat.cpp new file mode 100644 index 00000000000..8edca924cd7 --- /dev/null +++ b/src/util/serfloat.cpp @@ -0,0 +1,64 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include + +double DecodeDouble(uint64_t v) noexcept { + static constexpr double NANVAL = std::numeric_limits::quiet_NaN(); + static constexpr double INFVAL = std::numeric_limits::infinity(); + double sign = 1.0; + if (v & 0x8000000000000000) { + sign = -1.0; + v ^= 0x8000000000000000; + } + // Zero + if (v == 0) return copysign(0.0, sign); + // Infinity + if (v == 0x7ff0000000000000) return copysign(INFVAL, sign); + // Other numbers + int exp = (v & 0x7FF0000000000000) >> 52; + uint64_t man = v & 0xFFFFFFFFFFFFF; + if (exp == 2047) { + // NaN + return NANVAL; + } else if (exp == 0) { + // Subnormal + return copysign(ldexp((double)man, -1074), sign); + } else { + // Normal + return copysign(ldexp((double)(man + 0x10000000000000), -1075 + exp), sign); + } +} + +uint64_t EncodeDouble(double f) noexcept { + int cls = std::fpclassify(f); + uint64_t sign = 0; + if (copysign(1.0, f) == -1.0) { + f = -f; + sign = 0x8000000000000000; + } + // Zero + if (cls == FP_ZERO) return sign; + // Infinity + if (cls == FP_INFINITE) return sign | 0x7ff0000000000000; + // NaN + if (cls == FP_NAN) return 0x7ff8000000000000; + // Other numbers + int exp; + uint64_t man = std::round(std::frexp(f, &exp) * 9007199254740992.0); + if (exp < -1021) { + // Too small to represent, encode 0 + if (exp < -1084) return sign; + // Subnormal numbers + return sign | (man >> (-1021 - exp)); + } else { + // Too big to represent, encode infinity + if (exp > 1024) return sign | 0x7ff0000000000000; + // Normal numbers + return sign | (((uint64_t)(1022 + exp)) << 52) | (man & 0xFFFFFFFFFFFFF); + } +} diff --git a/src/util/serfloat.h b/src/util/serfloat.h new file mode 100644 index 00000000000..4d912b0176e --- /dev/null +++ b/src/util/serfloat.h @@ -0,0 +1,16 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SERFLOAT_H +#define BITCOIN_UTIL_SERFLOAT_H + +#include + +/* Encode a double using the IEEE 754 binary64 format. All NaNs are encoded as x86/ARM's + * positive quiet NaN with payload 0. */ +uint64_t EncodeDouble(double f) noexcept; +/* Reverse operation of DecodeDouble. DecodeDouble(EncodeDouble(f))==f unless isnan(f). */ +double DecodeDouble(uint64_t v) noexcept; + +#endif // BITCOIN_UTIL_SERFLOAT_H From bda33f98e2f32f2411fb0a8f5fb4f0a32abdf7d4 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 16 May 2021 17:46:26 -0700 Subject: [PATCH 3/6] Add unit tests for serfloat module --- src/Makefile.test.include | 1 + src/test/serfloat_tests.cpp | 92 +++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 src/test/serfloat_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index dc79ea3125a..e5f9c4cb529 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -121,6 +121,7 @@ BITCOIN_TESTS =\ test/script_tests.cpp \ test/script_standard_tests.cpp \ test/scriptnum_tests.cpp \ + test/serfloat_tests.cpp \ test/serialize_tests.cpp \ test/settings_tests.cpp \ test/sighash_tests.cpp \ diff --git a/src/test/serfloat_tests.cpp b/src/test/serfloat_tests.cpp new file mode 100644 index 00000000000..5b49488c5d4 --- /dev/null +++ b/src/test/serfloat_tests.cpp @@ -0,0 +1,92 @@ +// Copyright (c) 2014-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +#include + +#include +#include + +BOOST_FIXTURE_TEST_SUITE(serfloat_tests, BasicTestingSetup) + +namespace { + +uint64_t TestDouble(double f) { + uint64_t i = EncodeDouble(f); + double f2 = DecodeDouble(i); + if (std::isnan(f)) { + // NaN is not guaranteed to round-trip exactly. + BOOST_CHECK(std::isnan(f2)); + } else { + // Everything else is. + BOOST_CHECK(!std::isnan(f2)); + uint64_t i2 = EncodeDouble(f2); + BOOST_CHECK_EQUAL(f, f2); + BOOST_CHECK_EQUAL(i, i2); + } + return i; +} + +} // namespace + +BOOST_AUTO_TEST_CASE(double_serfloat_tests) { + BOOST_CHECK_EQUAL(TestDouble(0.0), 0); + BOOST_CHECK_EQUAL(TestDouble(-0.0), 0x8000000000000000); + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits::infinity()), 0x7ff0000000000000); + BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits::infinity()), 0xfff0000000000000); + + if (std::numeric_limits::is_iec559) { + BOOST_CHECK_EQUAL(sizeof(double), 8); + BOOST_CHECK_EQUAL(sizeof(uint64_t), 8); + // Test extreme values + TestDouble(std::numeric_limits::min()); + TestDouble(-std::numeric_limits::min()); + TestDouble(std::numeric_limits::max()); + TestDouble(-std::numeric_limits::max()); + TestDouble(std::numeric_limits::lowest()); + TestDouble(-std::numeric_limits::lowest()); + TestDouble(std::numeric_limits::quiet_NaN()); + TestDouble(-std::numeric_limits::quiet_NaN()); + TestDouble(std::numeric_limits::signaling_NaN()); + TestDouble(-std::numeric_limits::signaling_NaN()); + TestDouble(std::numeric_limits::denorm_min()); + TestDouble(-std::numeric_limits::denorm_min()); + // Test exact encoding: on currently supported platforms, EncodeDouble + // should produce exactly the same as the in-memory representation for non-NaN. + for (int j = 0; j < 1000; ++j) { + // Iterate over 9 specific bits exhaustively; the others are chosen randomly. + // These specific bits are the sign bit, and the 2 top and bottom bits of + // exponent and mantissa in the IEEE754 binary64 format. + for (int x = 0; x < 512; ++x) { + uint64_t v = InsecureRandBits(64); + v &= ~(uint64_t{1} << 0); + if (x & 1) v |= (uint64_t{1} << 0); + v &= ~(uint64_t{1} << 1); + if (x & 2) v |= (uint64_t{1} << 1); + v &= ~(uint64_t{1} << 50); + if (x & 4) v |= (uint64_t{1} << 50); + v &= ~(uint64_t{1} << 51); + if (x & 8) v |= (uint64_t{1} << 51); + v &= ~(uint64_t{1} << 52); + if (x & 16) v |= (uint64_t{1} << 52); + v &= ~(uint64_t{1} << 53); + if (x & 32) v |= (uint64_t{1} << 53); + v &= ~(uint64_t{1} << 61); + if (x & 64) v |= (uint64_t{1} << 61); + v &= ~(uint64_t{1} << 62); + if (x & 128) v |= (uint64_t{1} << 62); + v &= ~(uint64_t{1} << 63); + if (x & 256) v |= (uint64_t{1} << 63); + double f; + memcpy(&f, &v, 8); + uint64_t v2 = TestDouble(f); + if (!std::isnan(f)) BOOST_CHECK_EQUAL(v, v2); + } + } + } +} + +BOOST_AUTO_TEST_SUITE_END() From afd964d70b6f7583ecf89c380f80db07f5b66a60 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 16 May 2021 18:09:20 -0700 Subject: [PATCH 4/6] Convert existing float encoding tests --- src/test/serfloat_tests.cpp | 39 ++++++++++++++++++++++++++++++- src/test/serialize_tests.cpp | 45 ------------------------------------ 2 files changed, 38 insertions(+), 46 deletions(-) diff --git a/src/test/serfloat_tests.cpp b/src/test/serfloat_tests.cpp index 5b49488c5d4..54e07b0f61b 100644 --- a/src/test/serfloat_tests.cpp +++ b/src/test/serfloat_tests.cpp @@ -2,8 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include #include #include +#include +#include #include @@ -37,8 +40,14 @@ BOOST_AUTO_TEST_CASE(double_serfloat_tests) { BOOST_CHECK_EQUAL(TestDouble(-0.0), 0x8000000000000000); BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits::infinity()), 0x7ff0000000000000); BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits::infinity()), 0xfff0000000000000); + BOOST_CHECK_EQUAL(TestDouble(0.5), 0x3fe0000000000000ULL); + BOOST_CHECK_EQUAL(TestDouble(1.0), 0x3ff0000000000000ULL); + BOOST_CHECK_EQUAL(TestDouble(2.0), 0x4000000000000000ULL); + BOOST_CHECK_EQUAL(TestDouble(4.0), 0x4010000000000000ULL); + BOOST_CHECK_EQUAL(TestDouble(785.066650390625), 0x4088888880000000ULL); - if (std::numeric_limits::is_iec559) { + // Roundtrip test on IEC559-compatible systems + if (std::numeric_limits::is_iec559) { BOOST_CHECK_EQUAL(sizeof(double), 8); BOOST_CHECK_EQUAL(sizeof(uint64_t), 8); // Test extreme values @@ -89,4 +98,32 @@ BOOST_AUTO_TEST_CASE(double_serfloat_tests) { } } +/* +Python code to generate the below hashes: + + def reversed_hex(x): + return binascii.hexlify(''.join(reversed(x))) + def dsha256(x): + return hashlib.sha256(hashlib.sha256(x).digest()).digest() + + reversed_hex(dsha256(''.join(struct.pack('> val; + double j = DecodeDouble(val); + BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 0fdf4818236..0812ef643e2 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -88,51 +88,6 @@ BOOST_AUTO_TEST_CASE(sizes) BOOST_CHECK_EQUAL(GetSerializeSize(bool(0), 0), 1U); } -BOOST_AUTO_TEST_CASE(doubles_conversion) -{ - // Choose values that map unambiguously to binary floating point to avoid - // rounding issues at the compiler side. - BOOST_CHECK_EQUAL(ser_uint64_to_double(0x0000000000000000ULL), 0.0); - BOOST_CHECK_EQUAL(ser_uint64_to_double(0x3fe0000000000000ULL), 0.5); - BOOST_CHECK_EQUAL(ser_uint64_to_double(0x3ff0000000000000ULL), 1.0); - BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4000000000000000ULL), 2.0); - BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4010000000000000ULL), 4.0); - BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4088888880000000ULL), 785.066650390625); - - BOOST_CHECK_EQUAL(ser_double_to_uint64(0.0), 0x0000000000000000ULL); - BOOST_CHECK_EQUAL(ser_double_to_uint64(0.5), 0x3fe0000000000000ULL); - BOOST_CHECK_EQUAL(ser_double_to_uint64(1.0), 0x3ff0000000000000ULL); - BOOST_CHECK_EQUAL(ser_double_to_uint64(2.0), 0x4000000000000000ULL); - BOOST_CHECK_EQUAL(ser_double_to_uint64(4.0), 0x4010000000000000ULL); - BOOST_CHECK_EQUAL(ser_double_to_uint64(785.066650390625), 0x4088888880000000ULL); -} -/* -Python code to generate the below hashes: - - def reversed_hex(x): - return binascii.hexlify(''.join(reversed(x))) - def dsha256(x): - return hashlib.sha256(hashlib.sha256(x).digest()).digest() - - reversed_hex(dsha256(''.join(struct.pack('> j; - BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); - } -} - BOOST_AUTO_TEST_CASE(varints) { // encode From fff1cae43af959a601cf2558cb3c77f3c2b1aa80 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 18 May 2021 12:36:53 -0700 Subject: [PATCH 5/6] Convert uses of double-serialization to {En,De}codeDouble --- src/policy/fees.cpp | 44 ++++++++++++++++++++++++++++++----------- src/test/fuzz/float.cpp | 20 +++++++++++++------ 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 52c33621669..2ae5798ebec 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include static const char* FEE_ESTIMATES_FILENAME = "fee_estimates.dat"; @@ -26,6 +27,25 @@ std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon) assert(false); } +namespace { + +struct EncodedDoubleFormatter +{ + template void Ser(Stream &s, double v) + { + s << EncodeDouble(v); + } + + template void Unser(Stream& s, double& v) + { + uint64_t encoded; + s >> encoded; + v = DecodeDouble(encoded); + } +}; + +} // namespace + /** * We will instantiate an instance of this class to track transactions that were * included in a block. We will lump transactions into a bucket according to their @@ -356,12 +376,12 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, void TxConfirmStats::Write(CAutoFile& fileout) const { - fileout << decay; + fileout << Using(decay); fileout << scale; - fileout << m_feerate_avg; - fileout << txCtAvg; - fileout << confAvg; - fileout << failAvg; + fileout << Using>(m_feerate_avg); + fileout << Using>(txCtAvg); + fileout << Using>>(confAvg); + fileout << Using>>(failAvg); } void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) @@ -372,7 +392,7 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets size_t maxConfirms, maxPeriods; // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor - filein >> decay; + filein >> Using(decay); if (decay <= 0 || decay >= 1) { throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); } @@ -381,15 +401,15 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets throw std::runtime_error("Corrupt estimates file. Scale must be non-zero"); } - filein >> m_feerate_avg; + filein >> Using>(m_feerate_avg); if (m_feerate_avg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); } - filein >> txCtAvg; + filein >> Using>(txCtAvg); if (txCtAvg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); } - filein >> confAvg; + filein >> Using>>(confAvg); maxPeriods = confAvg.size(); maxConfirms = scale * maxPeriods; @@ -402,7 +422,7 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets } } - filein >> failAvg; + filein >> Using>>(failAvg); if (maxPeriods != failAvg.size()) { throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); } @@ -884,7 +904,7 @@ bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const else { fileout << historicalFirst << historicalBest; } - fileout << buckets; + fileout << Using>(buckets); feeStats->Write(fileout); shortStats->Write(fileout); longStats->Write(fileout); @@ -920,7 +940,7 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid"); } std::vector fileBuckets; - filein >> fileBuckets; + filein >> Using>(fileBuckets); size_t numBuckets = fileBuckets.size(); if (numBuckets <= 1 || numBuckets > 1000) { throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); diff --git a/src/test/fuzz/float.cpp b/src/test/fuzz/float.cpp index f806c82d14b..e95d593b227 100644 --- a/src/test/fuzz/float.cpp +++ b/src/test/fuzz/float.cpp @@ -7,10 +7,13 @@ #include #include #include +#include #include #include #include +#include +#include FUZZ_TARGET(float) { @@ -19,12 +22,17 @@ FUZZ_TARGET(float) { const double d = fuzzed_data_provider.ConsumeFloatingPoint(); (void)memusage::DynamicUsage(d); - assert(ser_uint64_to_double(ser_double_to_uint64(d)) == d); - CDataStream stream(SER_NETWORK, INIT_PROTO_VERSION); - stream << d; - double d_deserialized; - stream >> d_deserialized; - assert(d == d_deserialized); + uint64_t encoded = EncodeDouble(d); + if constexpr (std::numeric_limits::is_iec559) { + if (!std::isnan(d)) { + uint64_t encoded_in_memory; + std::copy((const unsigned char*)&d, (const unsigned char*)(&d + 1), (unsigned char*)&encoded_in_memory); + assert(encoded_in_memory == encoded); + } + } + double d_deserialized = DecodeDouble(encoded); + assert(std::isnan(d) == std::isnan(d_deserialized)); + assert(std::isnan(d) || d == d_deserialized); } } From 66545da2008cd9e806e41b74522ded259cd64f86 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 18 May 2021 12:37:02 -0700 Subject: [PATCH 6/6] Remove support for double serialization --- src/Makefile.am | 1 - src/compat/assumptions.h | 5 ----- src/serialize.h | 19 ------------------- src/test/fuzz/float.cpp | 3 --- src/test/fuzz/util.h | 2 -- src/test/serialize_tests.cpp | 2 -- src/util/types.h | 11 ----------- 7 files changed, 43 deletions(-) delete mode 100644 src/util/types.h diff --git a/src/Makefile.am b/src/Makefile.am index f3f58b41b7e..2c25f04d086 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -265,7 +265,6 @@ BITCOIN_CORE_H = \ util/tokenpipe.h \ util/trace.h \ util/translation.h \ - util/types.h \ util/ui_change_type.h \ util/url.h \ util/vector.h \ diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h index 5f50cde3ff6..7a254c3b67b 100644 --- a/src/compat/assumptions.h +++ b/src/compat/assumptions.h @@ -36,11 +36,6 @@ static_assert(std::numeric_limits::is_iec559, "IEEE 754 double assumed") // Example(s): Everywhere :-) static_assert(std::numeric_limits::digits == 8, "8-bit byte assumed"); -// Assumption: We assume floating-point widths. -// Example(s): Type punning in serialization code (ser_{float,double}_to_uint{32,64}). -static_assert(sizeof(float) == 4, "32-bit float assumed"); -static_assert(sizeof(double) == 8, "64-bit double assumed"); - // Assumption: We assume integer widths. // Example(s): GetSizeOfCompactSize and WriteCompactSize in the serialization // code. diff --git a/src/serialize.h b/src/serialize.h index 276f643c7f1..5ef846b9e91 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -23,7 +23,6 @@ #include #include -#include /** * The maximum size of a serialized object in bytes or number of elements @@ -123,20 +122,6 @@ template inline uint64_t ser_readdata64(Stream &s) s.read((char*)&obj, 8); return le64toh(obj); } -inline uint64_t ser_double_to_uint64(double x) -{ - uint64_t tmp; - std::memcpy(&tmp, &x, sizeof(x)); - static_assert(sizeof(tmp) == sizeof(x), "double and uint64_t assumed to have the same size"); - return tmp; -} -inline double ser_uint64_to_double(uint64_t y) -{ - double tmp; - std::memcpy(&tmp, &y, sizeof(y)); - static_assert(sizeof(tmp) == sizeof(y), "double and uint64_t assumed to have the same size"); - return tmp; -} ///////////////////////////////////////////////////////////////// @@ -221,8 +206,6 @@ template inline void Serialize(Stream& s, int32_t a ) { ser_wri template inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); } template inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); } template inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } -template inline void Serialize(Stream& s, float a ) { static_assert(ALWAYS_FALSE, "Not implemented"); } -template inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); } template inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); } template inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); } template inline void Serialize(Stream& s, const Span& span) { s.write(CharCast(span.data()), span.size()); } @@ -239,8 +222,6 @@ template inline void Unserialize(Stream& s, int32_t& a ) { a = template inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); } template inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); } template inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } -template inline void Unserialize(Stream& s, float& a ) { static_assert(ALWAYS_FALSE, "Not implemented"); } -template inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); } template inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); } template inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); } template inline void Unserialize(Stream& s, Span& span) { s.read(CharCast(span.data()), span.size()); } diff --git a/src/test/fuzz/float.cpp b/src/test/fuzz/float.cpp index e95d593b227..adef66a3ee0 100644 --- a/src/test/fuzz/float.cpp +++ b/src/test/fuzz/float.cpp @@ -3,15 +3,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include -#include #include #include #include #include #include -#include #include #include diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index ac8df782be8..cf7796e4ae4 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -513,7 +513,6 @@ void WriteToStream(FuzzedDataProvider& fuzzed_data_provider, Stream& stream) noe WRITE_TO_STREAM_CASE(uint32_t, fuzzed_data_provider.ConsumeIntegral()), WRITE_TO_STREAM_CASE(int64_t, fuzzed_data_provider.ConsumeIntegral()), WRITE_TO_STREAM_CASE(uint64_t, fuzzed_data_provider.ConsumeIntegral()), - WRITE_TO_STREAM_CASE(double, fuzzed_data_provider.ConsumeFloatingPoint()), WRITE_TO_STREAM_CASE(std::string, fuzzed_data_provider.ConsumeRandomLengthString(32)), WRITE_TO_STREAM_CASE(std::vector, ConsumeRandomLengthIntegralVector(fuzzed_data_provider))); } catch (const std::ios_base::failure&) { @@ -544,7 +543,6 @@ void ReadFromStream(FuzzedDataProvider& fuzzed_data_provider, Stream& stream) no READ_FROM_STREAM_CASE(uint32_t), READ_FROM_STREAM_CASE(int64_t), READ_FROM_STREAM_CASE(uint64_t), - READ_FROM_STREAM_CASE(double), READ_FROM_STREAM_CASE(std::string), READ_FROM_STREAM_CASE(std::vector)); } catch (const std::ios_base::failure&) { diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 0812ef643e2..58709178a49 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -70,7 +70,6 @@ BOOST_AUTO_TEST_CASE(sizes) BOOST_CHECK_EQUAL(sizeof(uint32_t), GetSerializeSize(uint32_t(0), 0)); BOOST_CHECK_EQUAL(sizeof(int64_t), GetSerializeSize(int64_t(0), 0)); BOOST_CHECK_EQUAL(sizeof(uint64_t), GetSerializeSize(uint64_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(double), GetSerializeSize(double(0), 0)); // Bool is serialized as char BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(bool(0), 0)); @@ -84,7 +83,6 @@ BOOST_AUTO_TEST_CASE(sizes) BOOST_CHECK_EQUAL(GetSerializeSize(uint32_t(0), 0), 4U); BOOST_CHECK_EQUAL(GetSerializeSize(int64_t(0), 0), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(uint64_t(0), 0), 8U); - BOOST_CHECK_EQUAL(GetSerializeSize(double(0), 0), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(bool(0), 0), 1U); } diff --git a/src/util/types.h b/src/util/types.h deleted file mode 100644 index 0047b00026b..00000000000 --- a/src/util/types.h +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_TYPES_H -#define BITCOIN_UTIL_TYPES_H - -template -inline constexpr bool ALWAYS_FALSE{false}; - -#endif // BITCOIN_UTIL_TYPES_H