diff --git a/src/Makefile.am b/src/Makefile.am index 6c9bfbf18..b5a0a0461 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -133,6 +133,7 @@ libmw_a_SOURCES = \ libmw/src/mmr/PMMRCache.cpp \ libmw/src/mmr/PMMR.cpp \ libmw/src/mmr/PruneList.cpp \ + libmw/src/mmr/Segment.cpp \ libmw/src/models/block/Block.cpp \ libmw/src/models/crypto/Commitment.cpp \ libmw/src/models/crypto/PublicKey.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 7d3123f76..a8d8edb9b 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -332,6 +332,7 @@ BITCOIN_TESTS += \ libmw/test/tests/mmr/Test_MMR.cpp \ libmw/test/tests/mmr/Test_MMRUtil.cpp \ libmw/test/tests/mmr/Test_PruneList.cpp \ + libmw/test/tests/mmr/Test_Segment.cpp \ libmw/test/tests/models/block/Test_Block.cpp \ libmw/test/tests/models/block/Test_Header.cpp \ libmw/test/tests/models/crypto/Test_BigInteger.cpp \ diff --git a/src/libmw/include/mw/mmr/MMRUtil.h b/src/libmw/include/mw/mmr/MMRUtil.h index 6d307afe7..e7678adf8 100644 --- a/src/libmw/include/mw/mmr/MMRUtil.h +++ b/src/libmw/include/mw/mmr/MMRUtil.h @@ -4,10 +4,13 @@ #include #include +class ILeafSet; + class MMRUtil { public: static mw::Hash CalcParentHash(const mmr::Index& index, const mw::Hash& left_hash, const mw::Hash& right_hash); + static std::vector CalcPeakIndices(const uint64_t num_nodes); static BitSet BuildCompactBitSet(const uint64_t num_leaves, const BitSet& unspent_leaf_indices); static BitSet DiffCompactBitSet(const BitSet& prev_compact, const BitSet& new_compact); diff --git a/src/libmw/include/mw/mmr/Segment.h b/src/libmw/include/mw/mmr/Segment.h new file mode 100644 index 000000000..3d99ae89e --- /dev/null +++ b/src/libmw/include/mw/mmr/Segment.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include +#include + +// Forward Declarations +class IMMR; +class ILeafSet; + +MMR_NAMESPACE + +/// +/// Represents a collection of contiguous, unpruned leaves, +/// and all hashes necessary to prove membership in the PMMR. +/// +struct Segment +{ + // The hashes of the requested unspent leaves, in ascending order + std::vector leaves; + + // The MMR node hashes needed to verify the integrity of the MMR & the provided leaves + std::vector hashes; + + // The "bagged" hash of the next lower peak (if there is one), + // which is necessary to compute the PMMR root + boost::optional lower_peak; +}; + +/// +/// Builds Segments for a provided MMR and segment. +/// +class SegmentFactory +{ +public: + static Segment Assemble( + const IMMR& mmr, + const ILeafSet& leafset, + const LeafIndex& first_leaf_idx, + const uint16_t num_leaves + ); + +private: + static std::set CalcHashIndices( + const ILeafSet& leafset, + const std::vector& peak_indices, + const mmr::LeafIndex& first_leaf_idx, + const mmr::LeafIndex& last_leaf_idx + ); + + static boost::optional BagNextLowerPeak( + const IMMR& mmr, + const std::vector& peak_indices, + const mmr::Index& peak_idx, + const uint64_t num_nodes + ); +}; + +END_NAMESPACE \ No newline at end of file diff --git a/src/libmw/include/mw/models/tx/UTXO.h b/src/libmw/include/mw/models/tx/UTXO.h index 8f7aad89e..d64d8bad1 100644 --- a/src/libmw/include/mw/models/tx/UTXO.h +++ b/src/libmw/include/mw/models/tx/UTXO.h @@ -21,8 +21,11 @@ public: const mw::Hash& GetOutputID() const noexcept { return m_output.GetOutputID(); } const Commitment& GetCommitment() const noexcept { return m_output.GetCommitment(); } + const PublicKey& GetSenderPubKey() const noexcept { return m_output.GetSenderPubKey(); } const PublicKey& GetReceiverPubKey() const noexcept { return m_output.GetReceiverPubKey(); } + const OutputMessage& GetOutputMessage() const noexcept { return m_output.GetOutputMessage(); } const RangeProof::CPtr& GetRangeProof() const noexcept { return m_output.GetRangeProof(); } + const Signature& GetSignature() const noexcept { return m_output.GetSignature(); } ProofData BuildProofData() const noexcept { return m_output.BuildProofData(); } IMPL_SERIALIZABLE(UTXO, obj) @@ -34,4 +37,44 @@ private: int32_t m_blockHeight; mmr::LeafIndex m_leafIdx; Output m_output; +}; + +/// +/// MWEB UTXO wrapper that supports serialization into multiple formats. +/// +class NetUTXO +{ +public: + static const uint8_t FULL_UTXO = 0x00; + static const uint8_t HASH_ONLY = 0x01; + static const uint8_t COMPACT_UTXO = 0x02; + + NetUTXO() = default; + NetUTXO(const uint8_t format, const UTXO::CPtr& utxo) + : m_format(format), m_utxo(utxo) { } + + template + inline void Serialize(Stream& s) const + { + s << VARINT(m_utxo->GetLeafIndex().Get()); + + if (m_format == FULL_UTXO) { + s << m_utxo->GetOutput(); + } else if (m_format == HASH_ONLY) { + s << m_utxo->GetOutputID(); + } else if (m_format == COMPACT_UTXO) { + s << m_utxo->GetCommitment(); + s << m_utxo->GetSenderPubKey(); + s << m_utxo->GetReceiverPubKey(); + s << m_utxo->GetOutputMessage(); + s << m_utxo->GetRangeProof()->GetHash(); + s << m_utxo->GetSignature(); + } else { + throw std::ios_base::failure("Unsupported MWEB UTXO serialization format"); + } + } + +private: + uint8_t m_format; + UTXO::CPtr m_utxo; }; \ No newline at end of file diff --git a/src/libmw/src/mmr/IMMR.cpp b/src/libmw/src/mmr/IMMR.cpp index a0b941000..996bd4201 100644 --- a/src/libmw/src/mmr/IMMR.cpp +++ b/src/libmw/src/mmr/IMMR.cpp @@ -6,32 +6,14 @@ using namespace mmr; mw::Hash IMMR::Root() const { const uint64_t num_nodes = mmr::LeafIndex::At(GetNumLeaves()).GetPosition(); - if (num_nodes == 0) { - return mw::Hash{}; - } // Find the "peaks" - std::vector peakIndices; - - uint64_t peakSize = BitUtil::FillOnesToRight(num_nodes); - uint64_t numLeft = num_nodes; - uint64_t sumPrevPeaks = 0; - while (peakSize != 0) { - if (numLeft >= peakSize) { - peakIndices.push_back(sumPrevPeaks + peakSize - 1); - sumPrevPeaks += peakSize; - numLeft -= peakSize; - } - - peakSize >>= 1; - } - - assert(numLeft == 0); + std::vector peak_indices = MMRUtil::CalcPeakIndices(num_nodes); // Bag 'em mw::Hash hash; - for (auto iter = peakIndices.crbegin(); iter != peakIndices.crend(); iter++) { - mw::Hash peakHash = GetHash(Index::At(*iter)); + for (auto iter = peak_indices.crbegin(); iter != peak_indices.crend(); iter++) { + mw::Hash peakHash = GetHash(*iter); if (hash.IsZero()) { hash = peakHash; } else { diff --git a/src/libmw/src/mmr/MMRUtil.cpp b/src/libmw/src/mmr/MMRUtil.cpp index fb64b01f4..72422852b 100644 --- a/src/libmw/src/mmr/MMRUtil.cpp +++ b/src/libmw/src/mmr/MMRUtil.cpp @@ -16,6 +16,32 @@ mw::Hash MMRUtil::CalcParentHash(const Index& index, const mw::Hash& left_hash, .hash(); } +std::vector MMRUtil::CalcPeakIndices(const uint64_t num_nodes) +{ + if (num_nodes == 0) { + return {}; + } + + // Find the "peaks" + std::vector peak_indices; + + uint64_t peakSize = BitUtil::FillOnesToRight(num_nodes); + uint64_t numLeft = num_nodes; + uint64_t sumPrevPeaks = 0; + while (peakSize != 0) { + if (numLeft >= peakSize) { + peak_indices.push_back(mmr::Index::At(sumPrevPeaks + peakSize - 1)); + sumPrevPeaks += peakSize; + numLeft -= peakSize; + } + + peakSize >>= 1; + } + + assert(numLeft == 0); + return peak_indices; +} + BitSet MMRUtil::BuildCompactBitSet(const uint64_t num_leaves, const BitSet& unspent_leaf_indices) { BitSet compactable_node_indices(num_leaves * 2); @@ -93,7 +119,7 @@ BitSet MMRUtil::CalcPrunedParents(const BitSet& unspent_leaf_indices) Index last_node = LeafIndex::At(unspent_leaf_indices.size()).GetNodeIndex(); uint64_t height = 1; - while ((std::pow(2, height + 1) - 2) <= last_node.GetPosition()) { + while ((uint64_t(2) << height) - 2 <= last_node.GetPosition()) { SiblingIter iter(height, last_node); while (iter.Next()) { Index right_child = iter.Get().GetRightChild(); diff --git a/src/libmw/src/mmr/Segment.cpp b/src/libmw/src/mmr/Segment.cpp new file mode 100644 index 000000000..6b31d31b7 --- /dev/null +++ b/src/libmw/src/mmr/Segment.cpp @@ -0,0 +1,147 @@ +#include +#include +#include +#include + +using namespace mmr; + +Segment SegmentFactory::Assemble(const IMMR& mmr, const ILeafSet& leafset, const LeafIndex& first_leaf_idx, const uint16_t num_leaves) +{ + if (!leafset.Contains(first_leaf_idx)) { + return {}; + } + + Segment segment; + segment.leaves.reserve(num_leaves); + + mmr::LeafIndex leaf_idx = first_leaf_idx; + mmr::LeafIndex last_leaf_idx = first_leaf_idx; + while (segment.leaves.size() < num_leaves && leaf_idx < leafset.GetNextLeafIdx()) { + if (leafset.Contains(leaf_idx)) { + last_leaf_idx = leaf_idx; + segment.leaves.push_back(mmr.GetHash(leaf_idx.GetNodeIndex())); + } + + ++leaf_idx; + } + + const uint64_t num_nodes = leafset.GetNextLeafIdx().GetPosition(); + std::vector peak_indices = MMRUtil::CalcPeakIndices(num_nodes); + assert(!peak_indices.empty()); + + // Populate hashes + std::set hash_indices = CalcHashIndices(leafset, peak_indices, first_leaf_idx, last_leaf_idx); + segment.hashes.reserve(hash_indices.size()); + for (const Index& idx : hash_indices) { + segment.hashes.push_back(mmr.GetHash(idx)); + } + + // Determine the lowest peak that can be calculated using the hashes we've already provided + auto peak = *std::find_if( + peak_indices.begin(), peak_indices.end(), + [&last_leaf_idx](const Index& peak_idx) { return peak_idx > last_leaf_idx.GetNodeIndex(); }); + + // Bag the next lower peak (if there is one), so the root can still be calculated + segment.lower_peak = BagNextLowerPeak(mmr, peak_indices, peak, num_nodes); + + return segment; +} + +std::set SegmentFactory::CalcHashIndices( + const ILeafSet& leafset, + const std::vector& peak_indices, + const mmr::LeafIndex& first_leaf_idx, + const mmr::LeafIndex& last_leaf_idx) +{ + std::set proof_indices; + + // 1. Add peaks of mountains to the left of first index + boost::optional prev_peak = boost::make_optional(false, Index()); + for (const Index& peak_idx : peak_indices) { + if (peak_idx < first_leaf_idx.GetNodeIndex()) { + proof_indices.insert(peak_idx); + prev_peak = peak_idx; + } else { + break; + } + } + + // 2. Add indices needed to reach left edge of mountain + auto on_mountain_left_edge = [prev_peak](const Index& idx) -> bool { + const uint64_t adjustment = !!prev_peak ? prev_peak->GetPosition() + 1 : 0; + return ((idx.GetPosition() + 2) - adjustment) == (uint64_t(2) << idx.GetHeight()); + }; + + Index idx = first_leaf_idx.GetNodeIndex(); + while (!on_mountain_left_edge(idx)) { + Index sibling_idx = idx.GetSibling(); + if (sibling_idx < idx) { + proof_indices.insert(sibling_idx); + idx = Index(idx.GetPosition() + 1, idx.GetHeight() + 1); + } else { + idx = Index(sibling_idx.GetPosition() + 1, sibling_idx.GetHeight() + 1); + } + } + + // 3. Add all pruned parents after first leaf and before last leaf + BitSet pruned_parents = MMRUtil::CalcPrunedParents(leafset.ToBitSet()); + for (uint64_t pos = first_leaf_idx.GetPosition(); pos < last_leaf_idx.GetPosition(); pos++) { + if (pruned_parents.test(pos)) { + proof_indices.insert(Index::At(pos)); + } + } + + // 4. Add indices needed to reach right edge of mountain containing the last leaf + auto peak_iter = std::find_if( + peak_indices.begin(), peak_indices.end(), + [&last_leaf_idx](const Index& peak_idx) { return peak_idx > last_leaf_idx.GetNodeIndex(); }); + assert(peak_iter != peak_indices.end()); + + Index peak = *peak_iter; + auto on_mountain_right_edge = [prev_peak, peak](const Index& idx) -> bool { + const uint64_t adjustment = prev_peak + .map([](const Index& i) { return i.GetPosition() + 1; }) + .value_or(0); + return (idx.GetPosition() - adjustment) >= ((peak.GetPosition() - adjustment) - peak.GetHeight()); + }; + + idx = last_leaf_idx.GetNodeIndex(); + while (!on_mountain_right_edge(idx)) { + Index sibling_idx = idx.GetSibling(); + if (sibling_idx > idx) { + proof_indices.insert(sibling_idx); + idx = Index(sibling_idx.GetPosition() + 1, idx.GetHeight() + 1); + } else { + idx = Index(idx.GetPosition() + 1, idx.GetHeight() + 1); + } + } + + return proof_indices; +} + +boost::optional SegmentFactory::BagNextLowerPeak( + const IMMR& mmr, + const std::vector& peak_indices, + const mmr::Index& peak_idx, + const uint64_t num_nodes) +{ + boost::optional lower_peak = boost::none; + + if (peak_idx != peak_indices.back()) { + // Bag peaks until we reach the next lowest + for (auto iter = peak_indices.crbegin(); iter != peak_indices.crend(); iter++) { + if (*iter == peak_idx) { + break; + } + + mw::Hash peakHash = mmr.GetHash(*iter); + if (lower_peak) { + lower_peak = MMRUtil::CalcParentHash(Index::At(num_nodes), peakHash, *lower_peak); + } else { + lower_peak = peakHash; + } + } + } + + return lower_peak; +} \ No newline at end of file diff --git a/src/libmw/test/tests/mmr/Test_MMRUtil.cpp b/src/libmw/test/tests/mmr/Test_MMRUtil.cpp index c987801ea..d0e00f22f 100644 --- a/src/libmw/test/tests/mmr/Test_MMRUtil.cpp +++ b/src/libmw/test/tests/mmr/Test_MMRUtil.cpp @@ -151,4 +151,14 @@ BOOST_AUTO_TEST_CASE(CalcPrunedParents) BOOST_REQUIRE(pruned_parent_hashes.str() == "0010100000000101000010000000100000000000000001001000000100000000000000000000000000000000000000000000"); } +BOOST_AUTO_TEST_CASE(CalcPeakIndices) +{ + std::vector peak_indices = MMRUtil::CalcPeakIndices(54); + BOOST_REQUIRE(peak_indices.size() == 4); + BOOST_REQUIRE(peak_indices[0] == 30); + BOOST_REQUIRE(peak_indices[1] == 45); + BOOST_REQUIRE(peak_indices[2] == 52); + BOOST_REQUIRE(peak_indices[3] == 53); +} + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/src/libmw/test/tests/mmr/Test_Segment.cpp b/src/libmw/test/tests/mmr/Test_Segment.cpp new file mode 100644 index 000000000..a6f6aaf33 --- /dev/null +++ b/src/libmw/test/tests/mmr/Test_Segment.cpp @@ -0,0 +1,116 @@ +// Copyright (c) 2022 The Litecoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include + +#include + +namespace std { +ostream& operator<<(ostream& os, const mw::Hash& hash) +{ + os << hash.ToHex(); + return os; +} +} // namespace std + +using namespace mmr; + +struct MMRWithLeafset { + IMMR::Ptr mmr; + ILeafSet::Ptr leafset; +}; + +static mmr::Leaf DeterministicLeaf(const size_t i) +{ + std::vector serialized{ + uint8_t(i >> 24), + uint8_t(i >> 16), + uint8_t(i >> 8), + uint8_t(i)}; + return mmr::Leaf::Create(mmr::LeafIndex::At(i), serialized); +} + +static MMRWithLeafset BuildDetermininisticMMR(const size_t num_leaves) +{ + auto mmr = std::make_shared(); + auto leafset = LeafSet::Open(GetDataDir(), 0); + for (size_t i = 0; i < num_leaves; i++) { + mmr->AddLeaf(DeterministicLeaf(i)); + leafset->Add(mmr::LeafIndex::At(i)); + } + + return MMRWithLeafset{mmr, leafset}; +} + +static boost::optional CalcBaggedPeak(const IMMR::Ptr& mmr, const mmr::Index& peak_idx) +{ + const uint64_t num_nodes = mmr->GetNextLeafIdx().GetPosition(); + + // Find the "peaks" + std::vector peak_indices = MMRUtil::CalcPeakIndices(num_nodes); + + // Bag 'em + boost::optional bagged_peak; + for (auto iter = peak_indices.crbegin(); iter != peak_indices.crend(); iter++) { + mw::Hash peakHash = mmr->GetHash(*iter); + if (bagged_peak) { + bagged_peak = MMRUtil::CalcParentHash(Index::At(num_nodes), peakHash, *bagged_peak); + } else { + bagged_peak = peakHash; + } + + BOOST_TEST_MESSAGE("peak(" << iter->GetPosition() << "): " << bagged_peak); + if (*iter == peak_idx) { + return bagged_peak; + } + } + + return bagged_peak; +} + +BOOST_FIXTURE_TEST_SUITE(TestSegment, MWEBTestingSetup) + +BOOST_AUTO_TEST_CASE(AssembleSegment) +{ + auto mmr_with_leafset = BuildDetermininisticMMR(15); + auto mmr = mmr_with_leafset.mmr; + auto leafset = mmr_with_leafset.leafset; + Segment segment = SegmentFactory::Assemble( + *mmr, + *leafset, + mmr::LeafIndex::At(0), + 4 + ); + + std::vector expected_leaves{ + DeterministicLeaf(0).GetHash(), + DeterministicLeaf(1).GetHash(), + DeterministicLeaf(2).GetHash(), + DeterministicLeaf(3).GetHash() + }; + BOOST_REQUIRE_EQUAL_COLLECTIONS(segment.leaves.begin(), segment.leaves.end(), expected_leaves.begin(), expected_leaves.end()); + + std::vector expected_hashes{ + mmr->GetHash(mmr::Index::At(13)) + }; + BOOST_REQUIRE_EQUAL_COLLECTIONS(segment.hashes.begin(), segment.hashes.end(), expected_hashes.begin(), expected_hashes.end()); + + boost::optional expected_lower_peak = CalcBaggedPeak(mmr, mmr::Index::At(21)); + BOOST_REQUIRE_EQUAL(expected_lower_peak, segment.lower_peak); + + // Verify PMMR root can be fully recomputed + mw::Hash n2 = MMRUtil::CalcParentHash(mmr::Index::At(2), segment.leaves[0], segment.leaves[1]); + mw::Hash n5 = MMRUtil::CalcParentHash(mmr::Index::At(5), segment.leaves[2], segment.leaves[3]); + mw::Hash n6 = MMRUtil::CalcParentHash(mmr::Index::At(6), n2, n5); + mw::Hash n14 = MMRUtil::CalcParentHash(mmr::Index::At(14), n6, segment.hashes[0]); + mw::Hash root = MMRUtil::CalcParentHash(Index::At(26), n14, *segment.lower_peak); + BOOST_REQUIRE_EQUAL(root, mmr->Root()); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 1855c6b79..fe2f6ca9c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,8 @@ static const int MAX_CMPCTBLOCK_DEPTH = 5; static const int MAX_BLOCKTXN_DEPTH = 10; /** Maximum depth of blocks we're willing to serve MWEB leafsets for. */ static const int MAX_MWEB_LEAFSET_DEPTH = 10; +/** Maximum number of MWEB UTXOs that can be requested in a batch. */ +static const uint16_t MAX_REQUESTED_MWEB_UTXOS = 4096; /** Size of the "block download window": how far ahead of our current height do we fetch? * Larger windows tolerate larger download speed differences between peer, but increase the potential * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably @@ -1718,34 +1721,39 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } } -class CMWEBLeafsetMsg +struct MWEBLeafsetMsg { -public: - CMWEBLeafsetMsg() = default; - CMWEBLeafsetMsg(uint256 block_hash_in, BitSet leafset_in) + MWEBLeafsetMsg() = default; + MWEBLeafsetMsg(uint256 block_hash_in, BitSet leafset_in) : block_hash(std::move(block_hash_in)), leafset(std::move(leafset_in)) { } - SERIALIZE_METHODS(CMWEBLeafsetMsg, obj) { READWRITE(obj.block_hash, obj.leafset); } + SERIALIZE_METHODS(MWEBLeafsetMsg, obj) { READWRITE(obj.block_hash, obj.leafset); } uint256 block_hash; BitSet leafset; }; -static void ProcessGetMWEBLeafset(CNode& pfrom, const CChainParams& chainparams, const CInv& inv, CConnman& connman) +static void ProcessGetMWEBLeafset(CNode& pfrom, const ChainstateManager& chainman, const CChainParams& chainparams, const CInv& inv, CConnman& connman) { ActivateBestChainIfNeeded(chainparams, inv); LOCK(cs_main); + if (chainman.ActiveChainstate().IsInitialBlockDownload()) { + LogPrint(BCLog::NET, "Ignoring mweb leafset request from peer=%d because node is in initial block download\n", pfrom.GetId()); + return; + } + CBlockIndex* pindex = LookupBlockIndex(inv.hash); - if (!pindex) { + if (!pindex || !chainman.ActiveChain().Contains(pindex)) { + LogPrint(BCLog::NET, "Ignoring mweb leafset request from peer=%d because requested block hash is not in active chain\n", pfrom.GetId()); return; } // TODO: Add an outbound limit - // Avoid leaking prune-height by never sending blocks below the NODE_NETWORK_LIMITED threshold - if (::ChainActive().Tip()->nHeight - pindex->nHeight > MAX_MWEB_LEAFSET_DEPTH) { - LogPrint(BCLog::NET, "Ignore block request below MAX_MWEB_LEAFSET_DEPTH threshold from peer=%d\n", pfrom.GetId()); + // For performance reasons, we limit how many blocks can be undone in order to rebuild the leafset + if (chainman.ActiveChain().Tip()->nHeight - pindex->nHeight > MAX_MWEB_LEAFSET_DEPTH) { + LogPrint(BCLog::NET, "Ignore mweb leafset request below MAX_MWEB_LEAFSET_DEPTH threshold from peer=%d\n", pfrom.GetId()); // disconnect node and prevent it from stalling (would otherwise wait for the MWEB leafset) if (!pfrom.HasPermission(PF_NOBAN)) { @@ -1755,24 +1763,167 @@ static void ProcessGetMWEBLeafset(CNode& pfrom, const CChainParams& chainparams, return; } - // Pruned nodes may have deleted the block, so check whether - // it's available before trying to send. - if (pindex->nStatus & BLOCK_HAVE_DATA && pindex->nStatus & BLOCK_HAVE_MWEB) { - // Rewind leafset to block height - BlockValidationState state; - CCoinsViewCache temp_view(&::ChainstateActive().CoinsTip()); - if (!ActivateArbitraryChain(state, temp_view, chainparams, pindex)) { + // Pruned nodes may have deleted the block, so check whether it's available before trying to send. + if (!(pindex->nStatus & BLOCK_HAVE_DATA) || !(pindex->nStatus & BLOCK_HAVE_MWEB)) { + LogPrint(BCLog::NET, "Ignoring mweb leafset request from peer=%d because block is either pruned or lacking mweb data\n", pfrom.GetId()); + + if (!pfrom.HasPermission(PF_NOBAN)) { + pfrom.fDisconnect = true; + } + return; + } + + // Rewind leafset to block height + BlockValidationState state; + CCoinsViewCache temp_view(&chainman.ActiveChainstate().CoinsTip()); + if (!ActivateArbitraryChain(state, temp_view, chainparams, pindex)) { + pfrom.fDisconnect = true; + return; + } + + // Serve leafset to peer + MWEBLeafsetMsg leafset_msg(pindex->GetBlockHash(), temp_view.GetMWEBCacheView()->GetLeafSet()->ToBitSet()); + connman.PushMessage(&pfrom, CNetMsgMaker(pfrom.GetCommonVersion()).Make(NetMsgType::MWEBLEAFSET, leafset_msg)); +} + +struct GetMWEBUTXOsMsg +{ + GetMWEBUTXOsMsg() = default; + + SERIALIZE_METHODS(GetMWEBUTXOsMsg, obj) + { + READWRITE(obj.block_hash, VARINT(obj.start_index), obj.num_requested, obj.output_format); + } + + uint256 block_hash; + uint64_t start_index; + uint16_t num_requested; + uint8_t output_format; +}; + +struct MWEBUTXOsMsg +{ + MWEBUTXOsMsg() = default; + + SERIALIZE_METHODS(MWEBUTXOsMsg, obj) + { + READWRITE(obj.block_hash, VARINT(obj.start_index), obj.output_format, obj.utxos, obj.proof_hashes); + } + + uint256 block_hash; + uint64_t start_index; + uint8_t output_format; + std::vector utxos; + std::vector proof_hashes; +}; + +static void ProcessGetMWEBUTXOs(CNode& pfrom, const ChainstateManager& chainman, const CChainParams& chainparams, CConnman& connman, const GetMWEBUTXOsMsg& get_utxos) +{ + if (get_utxos.num_requested > MAX_REQUESTED_MWEB_UTXOS) { + LogPrint(BCLog::NET, "getmwebutxos num_requested %u > %u, disconnect peer=%d\n", get_utxos.num_requested, MAX_REQUESTED_MWEB_UTXOS, pfrom.GetId()); + if (!pfrom.HasPermission(PF_NOBAN)) { + pfrom.fDisconnect = true; + } + return; + } + + static const std::set supported_formats{ + NetUTXO::HASH_ONLY, + NetUTXO::FULL_UTXO, + NetUTXO::COMPACT_UTXO}; + if (supported_formats.count(get_utxos.output_format) == 0) { + LogPrint(BCLog::NET, "getmwebutxos output_format %u not supported, disconnect peer=%d\n", get_utxos.output_format, pfrom.GetId()); + if (!pfrom.HasPermission(PF_NOBAN)) { + pfrom.fDisconnect = true; + } + return; + } + + LOCK(cs_main); + + if (chainman.ActiveChainstate().IsInitialBlockDownload()) { + LogPrint(BCLog::NET, "Ignoring getmwebutxos from peer=%d because node is in initial block download\n", pfrom.GetId()); + return; + } + + CBlockIndex* pindex = LookupBlockIndex(get_utxos.block_hash); + if (!pindex || !chainman.ActiveChain().Contains(pindex)) { + LogPrint(BCLog::NET, "Ignoring getmwebutxos from peer=%d because requested block hash is not in active chain\n", pfrom.GetId()); + return; + } + + // TODO: Add an outbound limit + + // For performance reasons, we limit how many blocks can be undone in order to rebuild the leafset + if (chainman.ActiveChain().Tip()->nHeight - pindex->nHeight > MAX_MWEB_LEAFSET_DEPTH) { + LogPrint(BCLog::NET, "Ignore getmwebutxos below MAX_MWEB_LEAFSET_DEPTH threshold from peer=%d\n", pfrom.GetId()); + + if (!pfrom.HasPermission(PF_NOBAN)) { + pfrom.fDisconnect = true; + } + + return; + } + + // Pruned nodes may have deleted the block, so check whether it's available before trying to send. + if (!(pindex->nStatus & BLOCK_HAVE_DATA) || !(pindex->nStatus & BLOCK_HAVE_MWEB)) { + LogPrint(BCLog::NET, "Ignoring getmwebutxos request from peer=%d because block is either pruned or lacking mweb data\n", pfrom.GetId()); + + if (!pfrom.HasPermission(PF_NOBAN)) { + pfrom.fDisconnect = true; + } + return; + } + + // Rewind leafset to block height + BlockValidationState state; + CCoinsViewCache temp_view(&chainman.ActiveChainstate().CoinsTip()); + if (!ActivateArbitraryChain(state, temp_view, chainparams, pindex)) { + pfrom.fDisconnect = true; + return; + } + + auto mweb_cache = temp_view.GetMWEBCacheView(); + auto pLeafset = mweb_cache->GetLeafSet(); + + mmr::Segment segment = mmr::SegmentFactory::Assemble( + *mweb_cache->GetOutputPMMR(), + *mweb_cache->GetLeafSet(), + mmr::LeafIndex::At(get_utxos.start_index), + get_utxos.num_requested + ); + if (segment.leaves.empty()) { + LogPrint(BCLog::NET, "Could not build segment requested by getmwebutxos from peer=%d\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + std::vector utxos; + utxos.reserve(segment.leaves.size()); + for (const mw::Hash& hash : segment.leaves) { + UTXO::CPtr utxo = mweb_cache->GetUTXO(hash); + if (!utxo) { + LogPrint(BCLog::NET, "Could not build segment requested by getmwebutxos from peer=%d\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } - // Serve leafset to peer - const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - auto pLeafset = temp_view.GetMWEBCacheView()->GetLeafSet(); - - CMWEBLeafsetMsg leafset_msg(pindex->GetBlockHash(), pLeafset->ToBitSet()); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::MWEBLEAFSET, leafset_msg)); + utxos.push_back(NetUTXO(get_utxos.output_format, utxo)); } + + std::vector proof_hashes = segment.hashes; + if (segment.lower_peak) { + proof_hashes.push_back(*segment.lower_peak); + } + + MWEBUTXOsMsg utxos_msg{ + get_utxos.block_hash, + get_utxos.start_index, + get_utxos.output_format, + std::move(utxos), + std::move(proof_hashes) + }; + connman.PushMessage(&pfrom, CNetMsgMaker(pfrom.GetCommonVersion()).Make(NetMsgType::MWEBUTXOS, utxos_msg)); } //! Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). @@ -1803,7 +1954,7 @@ static CTransactionRef FindTxForGetData(const CTxMemPool& mempool, const CNode& return {}; } -void static ProcessGetData(CNode& pfrom, Peer& peer, const CChainParams& chainparams, CConnman& connman, CTxMemPool& mempool, const std::atomic& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!cs_main, peer.m_getdata_requests_mutex) +void static ProcessGetData(CNode& pfrom, Peer& peer, const ChainstateManager& chainman, const CChainParams& chainparams, CConnman& connman, CTxMemPool& mempool, const std::atomic& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!cs_main, peer.m_getdata_requests_mutex) { AssertLockNotHeld(cs_main); @@ -1872,7 +2023,7 @@ void static ProcessGetData(CNode& pfrom, Peer& peer, const CChainParams& chainpa if (inv.IsGenBlkMsg()) { ProcessGetBlockData(pfrom, chainparams, inv, connman); } else if (inv.IsMsgMWEBLeafset()) { - ProcessGetMWEBLeafset(pfrom, chainparams, inv, connman); + ProcessGetMWEBLeafset(pfrom, chainman, chainparams, inv, connman); } // else: If the first item on the queue is an unknown type, we erase it // and continue processing the queue on the next call. @@ -2937,7 +3088,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat { LOCK(peer->m_getdata_requests_mutex); peer->m_getdata_requests.insert(peer->m_getdata_requests.end(), vInv.begin(), vInv.end()); - ProcessGetData(pfrom, *peer, m_chainparams, m_connman, m_mempool, interruptMsgProc); + ProcessGetData(pfrom, *peer, m_chainman, m_chainparams, m_connman, m_mempool, interruptMsgProc); } return; @@ -3970,6 +4121,13 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat return; } + if (msg_type == NetMsgType::GETMWEBUTXOS) { + GetMWEBUTXOsMsg get_utxos; + vRecv >> get_utxos; + ProcessGetMWEBUTXOs(pfrom, m_chainman, m_chainparams, m_connman, get_utxos); + return; + } + // Ignore unknown commands for extensibility LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(msg_type), pfrom.GetId()); return; @@ -4027,7 +4185,7 @@ bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic& interruptMsgP { LOCK(peer->m_getdata_requests_mutex); if (!peer->m_getdata_requests.empty()) { - ProcessGetData(*pfrom, *peer, m_chainparams, m_connman, m_mempool, interruptMsgProc); + ProcessGetData(*pfrom, *peer, m_chainman, m_chainparams, m_connman, m_mempool, interruptMsgProc); } } diff --git a/src/protocol.cpp b/src/protocol.cpp index 07e80d561..edb2c33f4 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -48,6 +48,8 @@ const char *CFCHECKPT="cfcheckpt"; const char *WTXIDRELAY="wtxidrelay"; const char *MWEBHEADER="mwebheader"; const char *MWEBLEAFSET="mwebleafset"; +const char *GETMWEBUTXOS="getmwebutxos"; +const char *MWEBUTXOS="mwebutxos"; } // namespace NetMsgType /** All known message types. Keep this in the same order as the list of @@ -181,7 +183,7 @@ std::string CInv::GetCommand() const case MSG_FILTERED_BLOCK: return cmd.append(NetMsgType::MERKLEBLOCK); case MSG_CMPCT_BLOCK: return cmd.append(NetMsgType::CMPCTBLOCK); case MSG_MWEB_HEADER: return cmd.append(NetMsgType::MWEBHEADER); - case MSG_MWEB_LEAFSET: return cmd.append(NetMsgType::MWEBLEAFSET); + case MSG_MWEB_LEAFSET: return cmd.append(NetMsgType::MWEBLEAFSET); default: throw std::out_of_range(strprintf("CInv::GetCommand(): type=%d unknown type", type)); } diff --git a/src/protocol.h b/src/protocol.h index 8ea2b9d7f..5323338b5 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -264,16 +264,28 @@ extern const char* WTXIDRELAY; * Contains a CMerkleBlockWithMWEB. * Sent in response to a getdata message which requested a * block using the inventory type MSG_MWEB_HEADER. - * @since protocol version 70017 as described by LIP-0007 + * @since protocol version 70017 as described by LIP-0006 */ extern const char* MWEBHEADER; /** * Contains a block hash and its serialized leafset. * Sent in response to a getdata message which requested * data using the inventory type MSG_MWEB_LEAFSET. - * @since protocol version 70017 as described by LIP-0007 + * @since protocol version 70017 as described by LIP-0006 */ extern const char* MWEBLEAFSET; +/** + * getmwebutxos requests a variable number of consecutive + * MWEB utxos at the time of the provided block hash. + * @since protocol version 70017 as described by LIP-0006 + */ +extern const char* GETMWEBUTXOS; +/** + * Contains a list of MWEB UTXOs that were requested in + * a getmwebutxos message. + * @since protocol version 70017 as described by LIP-0006 + */ +extern const char* MWEBUTXOS; }; // namespace NetMsgType /* Get a vector of all valid message types (see above) */ @@ -441,8 +453,8 @@ enum GetDataMsg : uint32_t { // MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG, MSG_MWEB_BLOCK = MSG_WITNESS_BLOCK | MSG_MWEB_FLAG, MSG_MWEB_TX = MSG_WITNESS_TX | MSG_MWEB_FLAG, - MSG_MWEB_HEADER = 8 | MSG_MWEB_FLAG, //!< Defined in LIP-0007 - MSG_MWEB_LEAFSET = 9 | MSG_MWEB_FLAG, //!< Defined in LIP-0007 + MSG_MWEB_HEADER = 8 | MSG_MWEB_FLAG, //!< Defined in LIP-0006 + MSG_MWEB_LEAFSET = 9 | MSG_MWEB_FLAG, //!< Defined in LIP-0006 }; /** inv message data */ diff --git a/test/functional/mweb_p2p.py b/test/functional/mweb_p2p.py index f5a117ff6..aa46f4b64 100644 --- a/test/functional/mweb_p2p.py +++ b/test/functional/mweb_p2p.py @@ -3,7 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ -Test LIP-0007 +Test LIP-0006 1. Test getdata 'mwebheader' *before* MWEB activation 2. Test getdata 'mwebheader' *after* MWEB activation diff --git a/test/functional/test_framework/ltc_util.py b/test/functional/test_framework/ltc_util.py index 95c46451b..97553cab4 100644 --- a/test/functional/test_framework/ltc_util.py +++ b/test/functional/test_framework/ltc_util.py @@ -126,7 +126,7 @@ def get_mweb_header(node, block_hash = None): # TODO: In the future, this should support passing in pegins and pegouts. node - The node to use to lookup the latest block. -mweb_hash - The block to retrieve the MWEB header for. If not provided, the best block hash will be used. +mweb_hash - The hash of the MWEB to commit to. Returns the built HogEx transaction as a 'CTransaction' """ diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 9a4b37039..be94edebf 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1951,7 +1951,7 @@ class Hash: return self.serialize().hex() def to_byte_arr(self): - return hex_str_to_bytes(self.to_hex()) + return self.serialize() @classmethod def deserialize(cls, f): @@ -2111,7 +2111,7 @@ class MWEBOutputMessage: r += struct.pack(" 50 else "") \ No newline at end of file + return "msg_mwebleafset(block_hash=%s, leafset=%s%s)" % (repr(self.block_hash), repr(leafset_hex)[:50], "..." if len(leafset_hex) > 50 else "") + +class msg_getmwebutxos: + __slots__ = ("block_hash", "start_index", "num_requested", "output_format") + msgtype = b"getmwebutxos" + + def __init__(self, block_hash=None, start_index=0, num_requested=0, output_format=0): + self.block_hash = block_hash + self.start_index = start_index + self.num_requested = num_requested + self.output_format = output_format + + def deserialize(self, f): + self.block_hash = Hash.deserialize(f) + self.start_index = deser_varint(f) + self.num_requested = (struct.unpack("> 8) + struct.pack("B", self.num_requested & 0xFF) + r += struct.pack("B", self.output_format) + return r + + def __repr__(self): + return ("msg_getmwebutxos(block_hash=%s, start_index=%d, num_requested=%d, output_format=%d)" % + (repr(self.block_hash), self.start_index, self.num_requested, self.output_format)) + + +class msg_mwebutxos: + __slots__ = ("block_hash", "start_index", "output_format", "utxos", "proof_hashes") + msgtype = b"mwebutxos" + + def __init__(self, block_hash=None, start_index=0, output_format=0, utxos=None, proof_hashes=None): + self.block_hash = block_hash + self.start_index = start_index + self.output_format = output_format + self.utxos = utxos + self.proof_hashes = proof_hashes + + def deserialize(self, f): + self.block_hash = Hash.deserialize(f) + self.start_index = deser_varint(f) + self.output_format = struct.unpack("B", f.read(1))[0] + + if self.output_format == 0: + self.utxos = deser_vector(f, Hash) + elif self.output_format == 1: + self.utxos = deser_vector(f, MWEBOutput) + else: + self.utxos = deser_vector(f, MWEBCompactOutput) + + self.proof_hashes = deser_vector(f, Hash) + + def serialize(self): + r = b"" + r += self.block_hash.serialize() + r += ser_varint(self.start_index) + r += struct.pack("B", self.output_format) + r += ser_vector(self.utxos) + r += ser_vector(self.proof_hashes) + return r + + def __repr__(self): + return ("msg_mwebutxos(block_hash=%s, start_index=%d, output_format=%d, utxos=%s, proof_hashes=%s)" % + (repr(self.block_hash), self.start_index, self.output_format, repr(self.utxos), repr(self.proof_hashes))) \ No newline at end of file diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index b80b66b16..b1a3ef9d9 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -51,12 +51,14 @@ from test_framework.messages import ( msg_getblocktxn, msg_getdata, msg_getheaders, + msg_getmwebutxos, msg_headers, msg_inv, msg_mempool, msg_merkleblock, msg_mwebheader, msg_mwebleafset, + msg_mwebutxos, msg_notfound, msg_ping, msg_pong, @@ -101,12 +103,14 @@ MESSAGEMAP = { b"getblocktxn": msg_getblocktxn, b"getdata": msg_getdata, b"getheaders": msg_getheaders, + b"getmwebutxos": msg_getmwebutxos, b"headers": msg_headers, b"inv": msg_inv, b"mempool": msg_mempool, b"merkleblock": msg_merkleblock, b"mwebheader": msg_mwebheader, b"mwebleafset": msg_mwebleafset, + b"mwebutxos": msg_mwebutxos, b"notfound": msg_notfound, b"ping": msg_ping, b"pong": msg_pong,