diff --git a/src/index/base.h b/src/index/base.h index 8cb8ad8effe..d8fd8566933 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -122,9 +122,6 @@ protected: void ChainStateFlushed(const kernel::ChainstateRole& role, const CBlockLocator& locator) override; - /// Return custom notification options for index. - [[nodiscard]] virtual interfaces::Chain::NotifyOptions CustomOptions() { return {}; } - /// Initialize internal state from the database and block index. [[nodiscard]] virtual bool CustomInit(const std::optional& block) { return true; } @@ -151,6 +148,9 @@ public: /// Get the name of the index for display in logs. const std::string& GetName() const LIFETIMEBOUND { return m_name; } + /// Return custom notification options for index. + [[nodiscard]] virtual interfaces::Chain::NotifyOptions CustomOptions() { return {}; } + /// Blocks the current thread until the index is caught up to the current /// state of the block chain. This only blocks if the index has gotten in /// sync once and only needs to process blocks in the ValidationInterface diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 96d393a3a97..0bb4a74e125 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -63,8 +63,6 @@ private: std::optional ReadFilterHeader(int height, const uint256& expected_block_hash); protected: - interfaces::Chain::NotifyOptions CustomOptions() override; - bool CustomInit(const std::optional& block) override; bool CustomCommit(CDBBatch& batch) override; @@ -80,6 +78,8 @@ public: explicit BlockFilterIndex(std::unique_ptr chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + interfaces::Chain::NotifyOptions CustomOptions() override; + BlockFilterType GetFilterType() const { return m_filter_type; } /** Get a single filter by block. */ diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index 041c0b896a6..0e26fba56d9 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -52,8 +52,6 @@ private: bool AllowPrune() const override { return true; } protected: - interfaces::Chain::NotifyOptions CustomOptions() override; - bool CustomInit(const std::optional& block) override; bool CustomCommit(CDBBatch& batch) override; @@ -68,6 +66,8 @@ public: // Constructs the index, which becomes available to be queried. explicit CoinStatsIndex(std::unique_ptr chain, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + interfaces::Chain::NotifyOptions CustomOptions() override; + // Look up stats for a specific block using CBlockIndex std::optional LookUpStats(const CBlockIndex& block_index) const; }; diff --git a/src/init.cpp b/src/init.cpp index 841fdec5b2c..02de132a3ad 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -2265,41 +2265,70 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) bool StartIndexBackgroundSync(NodeContext& node) { - // Find the oldest block among all indexes. - // This block is used to verify that we have the required blocks' data stored on disk, - // starting from that point up to the current tip. - // indexes_start_block='nullptr' means "start from height 0". - std::optional indexes_start_block; - std::string older_index_name; ChainstateManager& chainman = *Assert(node.chainman); const Chainstate& chainstate = WITH_LOCK(::cs_main, return chainman.ValidatedChainstate()); const CChain& index_chain = chainstate.m_chain; + const int current_height = WITH_LOCK(::cs_main, return index_chain.Height()); - for (auto index : node.indexes) { - const IndexSummary& summary = index->GetSummary(); - if (summary.synced) continue; + // Skip checking data availability if we have not synced any blocks yet + if (current_height > 0) { + // Before starting index sync, verify that all required block data is available + // on disk from each index's current sync position up to the chain tip. + // + // This is done separately for undo and block data: First we verify block + undo + // data existence from tip down to the lowest height required by any index that + // needs undo data (e.g., coinstatsindex, blockfilterindex). Then, if any + // block-only index needs to sync from a lower height than previously covered, + // verify block data existence down to that lower height. + // + // This avoids checking undo data for blocks where no index requires it, + // though currently block and undo data availability are synchronized on disk + // under normal circumstances. + std::optional block_start; + std::string block_start_name; + std::optional undo_start; + std::string undo_start_name; - // Get the last common block between the index best block and the active chain - LOCK(::cs_main); - const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(summary.best_block_hash); - if (!index_chain.Contains(pindex)) { - pindex = index_chain.FindFork(pindex); + for (const auto& index : node.indexes) { + const IndexSummary& summary = index->GetSummary(); + if (summary.synced) continue; + + // Get the last common block between the index best block and the active chain + const CBlockIndex* pindex = nullptr; + { + LOCK(::cs_main); + pindex = chainman.m_blockman.LookupBlockIndex(summary.best_block_hash); + if (!index_chain.Contains(pindex)) { + pindex = index_chain.FindFork(pindex); + } + } + if (!pindex) { + pindex = index_chain.Genesis(); + } + + bool need_undo = index->CustomOptions().connect_undo_data; + auto& op_start_index = need_undo ? undo_start : block_start; + auto& name_index = need_undo ? undo_start_name : block_start_name; + + if (op_start_index && pindex->nHeight >= op_start_index.value()->nHeight) continue; + op_start_index = pindex; + name_index = summary.name; } - if (!indexes_start_block || !pindex || pindex->nHeight < indexes_start_block.value()->nHeight) { - indexes_start_block = pindex; - older_index_name = summary.name; - if (!pindex) break; // Starting from genesis so no need to look for earlier block. + // Verify all blocks needed to sync to current tip are present including undo data. + if (undo_start) { + LOCK(::cs_main); + if (!chainman.m_blockman.CheckBlockDataAvailability(*index_chain.Tip(), *Assert(undo_start.value()), BlockStatus{BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO})) { + return InitError(Untranslated(strprintf("%s best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)", undo_start_name))); + } } - }; - // Verify all blocks needed to sync to current tip are present. - if (indexes_start_block) { - LOCK(::cs_main); - const CBlockIndex* start_block = *indexes_start_block; - if (!start_block) start_block = chainman.ActiveChain().Genesis(); - if (!chainman.m_blockman.CheckBlockDataAvailability(*index_chain.Tip(), *Assert(start_block))) { - return InitError(Untranslated(strprintf("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", older_index_name))); + // Verify all blocks needed to sync to current tip are present unless we already checked all of them above. + if (block_start && !(undo_start && undo_start.value()->nHeight <= block_start.value()->nHeight)) { + LOCK(::cs_main); + if (!chainman.m_blockman.CheckBlockDataAvailability(*index_chain.Tip(), *Assert(block_start.value()), BlockStatus{BLOCK_HAVE_DATA})) { + return InitError(Untranslated(strprintf("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", block_start_name))); + } } } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 8cb398fea64..90bae4db8ae 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -411,7 +411,8 @@ public: /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); - //! Check if all blocks in the [upper_block, lower_block] range have data available. + //! Check if all blocks in the [upper_block, lower_block] range have data available as + //! defined by the status mask. //! The caller is responsible for ensuring that lower_block is an ancestor of upper_block //! (part of the same chain). bool CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block, BlockStatus block_status = BLOCK_HAVE_DATA) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index ff99a1ee4ff..f1b6c017ad8 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -149,6 +149,7 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_availability, TestChain100Setup) // detect this by using a status mask. first_available_block->nStatus &= ~BLOCK_HAVE_UNDO; BOOST_CHECK(!blockman.CheckBlockDataAvailability(tip, *first_available_block, BlockStatus{BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO})); + BOOST_CHECK(blockman.CheckBlockDataAvailability(tip, *first_available_block, BlockStatus{BLOCK_HAVE_DATA})); } BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_part, TestChain100Setup) diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py index 81c75f2f02f..f552ee89788 100755 --- a/test/functional/feature_index_prune.py +++ b/test/functional/feature_index_prune.py @@ -130,8 +130,8 @@ class FeatureIndexPruneTest(BitcoinTestFramework): self.stop_node(i) self.log.info("make sure we get an init error when starting the nodes again with the indices") - filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" - stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)" + stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)" end_msg = f"{os.linesep}Error: A fatal internal error occurred, see debug.log for details: Failed to start indexes, shutting down…" for i, msg in enumerate([filter_msg, stats_msg, filter_msg]): self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg)