diff --git a/CMakeLists.txt b/CMakeLists.txt index f992a8d6af4..4c0f4bad810 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,6 +126,8 @@ if(WITH_ZMQ) find_package(ZeroMQ 4.0.0 MODULE REQUIRED) endif() +option(WITH_EMBEDDED_ASMAP "Embed default ASMap data." ON) + option(WITH_USDT "Enable tracepoints for Userspace, Statically Defined Tracing." OFF) if(WITH_USDT) find_package(USDT MODULE REQUIRED) @@ -215,6 +217,7 @@ if(BUILD_FOR_FUZZING) set(BUILD_GUI OFF) set(ENABLE_EXTERNAL_SIGNER OFF) set(WITH_ZMQ OFF) + set(WITH_EMBEDDED_ASMAP OFF) set(BUILD_TESTS OFF) set(BUILD_GUI_TESTS OFF) set(BUILD_BENCH OFF) @@ -653,6 +656,7 @@ else() set(ipc_status OFF) endif() message(" IPC ................................. ${ipc_status}") +message(" Embedded ASMap ...................... ${WITH_EMBEDDED_ASMAP}") message(" USDT tracing ........................ ${WITH_USDT}") message(" QR code (GUI) ....................... ${WITH_QRENCODE}") message(" DBus (GUI) .......................... ${WITH_DBUS}") diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 28446a705dc..0b21ab226a9 100755 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -17,4 +17,5 @@ export BITCOIN_CONFIG="\ --preset=dev-mode \ -DREDUCE_EXPORTS=ON \ -DENABLE_WALLET=OFF \ + -DWITH_EMBEDDED_ASMAP=OFF \ " diff --git a/cmake/script/GenerateHeaderFromRaw.cmake b/cmake/script/GenerateHeaderFromRaw.cmake index d373d1c4f87..2c40e419f60 100644 --- a/cmake/script/GenerateHeaderFromRaw.cmake +++ b/cmake/script/GenerateHeaderFromRaw.cmake @@ -18,6 +18,5 @@ ${formatted_bytes} }; inline constexpr std::span ${raw_source_basename}{detail_${raw_source_basename}_raw}; -} -") +}") file(WRITE ${HEADER_PATH} "${header_content}") diff --git a/contrib/README.md b/contrib/README.md index f23d7ac557b..037ea2f0690 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -18,6 +18,9 @@ A Linux bash script that will set up traffic control (tc) to limit the outgoing ### [Seeds](/contrib/seeds) ### Utility to generate the pnSeed[] array that is compiled into the client. +### [ASMap](/contrib/asmap) ### +Utilities to analyze and process asmap files. + Build Tools and Keys --------------------- diff --git a/doc/asmap-data.md b/doc/asmap-data.md new file mode 100644 index 00000000000..09e2f95c974 --- /dev/null +++ b/doc/asmap-data.md @@ -0,0 +1,59 @@ +# Embedded ASMap data + +## Background + +The ASMap feature (available via `-asmap`) makes it possible to use a peer's AS Number (ASN), an ISP/hoster identifier, +in netgroup bucketing in order to ensure a higher diversity in the peer +set. When not using this, the default behavior is to have the buckets formed +based on IP prefixes but this does not +prevent having connections dominated by peers at the same large-scale hoster, +for example, since such companies usually control many diverse IP ranges. +In order to use ASMap, the mapping between IP prefixes and AS Numbers needs +to be available. This mapping data can be provided through an external file +but Bitcoin Core also embeds a default map in its builds to make the feature +available to users when they are unable to provide a file. + +## Data sourcing and tools + +ASMap is a mapping of IP prefix to ASN, essentially a snapshot of the +internet routing table at some point in time. Due to the high volatility +of parts of this routing table and the known vulnerabilities in the BGP +protocol it is challenging to collect this data and prove its consistency. +Sourcing the data from a single trusted source is problematic as well. + +The [Kartograf](https://github.com/asmap/kartograf) tool was created to +deal with these uncertainties as good as possible. The mapping data is sourced from RPKI, IRR and +Routeviews. The former two are themselves used as security mechanisms to +protect against BGP security issues, which is why they are considered more secure and +their data takes precedence. The latter is a trusted collector of BGP traffic +and only used for IP space that is not covered by RPKI and IRR. + +The process in which the Kartograf project parses, processes and merges these +data sources is deterministic. Given the raw download files from these +different sources, anyone can build their own map file and verify the content +matches with other users' results. Before the map is usable by Bitcoin Core +it needs to be encoded as well. This is done using `asmap-tool.py` in `contrib/asmap` +and this step is deterministic as well. + +When it comes to obtaining the initial input data, the high volatility remains +a challenge if users don't want to trust a single creator of the used ASMap file. +To overcome this, multiple users can start the download process at the exact +same time which leads to a high likelihood that their downloaded data will be +similar enough that they receive the same output at the end of the process. +This process is regularly coordinated at the [asmap-data](https://github.com/asmap/asmap-data) +project. If enough participants have joined the effort (5 or more is recommended) and a majority of the +participants have received the same result, the resulting ASMap file is added +to the repository for public use. Files will not be merged to the repository +without at least two additional reviewers confirming that the process described +above was followed as expected and that the encoding step yielded the same +file hash. New files are created on an ongoing basis but without any central planning +or an explicit schedule. + +## Release process + +As an upcoming release approaches the embedded ASMap data should be updated +by replacing the `ip_asn.dat` with a newer ASMap file from the asmap-data +repository so that its data is embedded in the release. Ideally, there may be a file +already created recently that can be selected for an upcoming release. Alternatively, +a new creation process can be initiated with the goal of obtaining a fresh map +for use in the upcoming release. diff --git a/doc/release-process.md b/doc/release-process.md index 272f36eadcf..90ffd852e1a 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -30,6 +30,7 @@ Release Process * Update translations see [translation_process.md](/doc/translation_process.md#synchronising-translations). * Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/27488) for an example. +* Update embedded asmap data at `/src/node/data/ip_asn.dat`, see [asmap data documentation](./asmap-data.md). * Update the following variables in [`src/kernel/chainparams.cpp`](/src/kernel/chainparams.cpp) for mainnet, testnet, and signet: - `m_assumed_blockchain_size` and `m_assumed_chain_state_size` with the current size plus some overhead (see [this](#how-to-calculate-assumed-blockchain-and-chain-state-size) for information on how to calculate them). diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cf1f26c9f24..2d64ee88fb4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -286,6 +286,13 @@ target_link_libraries(bitcoin_node $ $ ) +if(WITH_EMBEDDED_ASMAP) + target_compile_definitions(bitcoin_node PRIVATE ENABLE_EMBEDDED_ASMAP=1) + include(TargetDataSources) + target_raw_data_sources(bitcoin_node NAMESPACE node::data + node/data/ip_asn.dat + ) +endif() # Bitcoin wrapper executable that can call other executables. if(BUILD_BITCOIN_BIN) diff --git a/src/init.cpp b/src/init.cpp index e6cc2045b4a..e2cdb9c6477 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -119,6 +119,10 @@ #include #endif +#ifdef ENABLE_EMBEDDED_ASMAP +#include +#endif + using common::AmountErrMsg; using common::InvalidPortErrMsg; using common::ResolveErrMsg; @@ -530,7 +534,13 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-addnode=", strprintf("Add a node to connect to and attempt to keep the connection open (see the addnode RPC help for more info). This option can be specified multiple times to add multiple nodes; connections are limited to %u at a time and are counted separately from the -maxconnections limit.", MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); - argsman.AddArg("-asmap=", "Specify asn mapping used for bucketing of the peers. Relative paths will be prefixed by the net-specific datadir location.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-asmap=", strprintf("Specify asn mapping used for bucketing of the peers. Relative paths will be prefixed by the net-specific datadir location.%s", + #ifdef ENABLE_EMBEDDED_ASMAP + " If a bool arg is given (-asmap or -asmap=1), the embedded mapping data in the binary will be used." + #else + "" + #endif + ), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bantime=", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bind=[:][=onion]", strprintf("Bind to given address and always listen on it (default: 0.0.0.0). Use [host]:port notation for IPv6. Append =onion to tag any incoming connections to that address and port as incoming Tor connections (default: 127.0.0.1:%u=onion, testnet3: 127.0.0.1:%u=onion, testnet4: 127.0.0.1:%u=onion, signet: 127.0.0.1:%u=onion, regtest: 127.0.0.1:%u=onion)", defaultChainParams->GetDefaultPort() + 1, testnetChainParams->GetDefaultPort() + 1, testnet4ChainParams->GetDefaultPort() + 1, signetChainParams->GetDefaultPort() + 1, regtestChainParams->GetDefaultPort() + 1), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-cjdnsreachable", "If set, then this host is configured for CJDNS (connecting to fc00::/8 addresses would lead us to the CJDNS network, see doc/cjdns.md) (default: 0)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1560,29 +1570,50 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) ApplyArgsManOptions(args, peerman_opts); { - // Read asmap file if configured and initialize + // Read asmap file if configured or embedded asmap data and initialize // Netgroupman with or without it assert(!node.netgroupman); if (args.IsArgSet("-asmap") && !args.IsArgNegated("-asmap")) { - fs::path asmap_path = args.GetPathArg("-asmap"); - if (asmap_path.empty()) { - InitError(_("-asmap requires a file path. Use -asmap=.")); - return false; + uint256 asmap_version{}; + if (!args.GetBoolArg("-asmap", false)) { + fs::path asmap_path = args.GetPathArg("-asmap"); + if (!asmap_path.is_absolute()) { + asmap_path = args.GetDataDirNet() / asmap_path; + } + + // If a specific path was passed with the asmap argument check if + // the file actually exists in that location + if (!fs::exists(asmap_path)) { + InitError(strprintf(_("Could not find asmap file %s"), fs::quoted(fs::PathToString(asmap_path)))); + return false; + } + + // If a file exists at the path, try to read the file + std::vector asmap{DecodeAsmap(asmap_path)}; + if (asmap.empty()) { + InitError(strprintf(_("Could not parse asmap file %s"), fs::quoted(fs::PathToString(asmap_path)))); + return false; + } + asmap_version = AsmapVersion(asmap); + node.netgroupman = std::make_unique(NetGroupManager::WithLoadedAsmap(std::move(asmap))); + } else { + #ifdef ENABLE_EMBEDDED_ASMAP + // Use the embedded asmap data + std::span asmap{node::data::ip_asn}; + if (asmap.empty() || !CheckStandardAsmap(asmap)) { + InitError(strprintf(_("Could not read embedded asmap data"))); + return false; + } + node.netgroupman = std::make_unique(NetGroupManager::WithEmbeddedAsmap(asmap)); + asmap_version = AsmapVersion(asmap); + LogInfo("Opened asmap data (%zu bytes) from embedded byte array\n", asmap.size()); + #else + // If there is no embedded data, fail and report it since + // the user tried to use it + InitError(strprintf(_("Embedded asmap data not available"))); + return false; + #endif } - if (!asmap_path.is_absolute()) { - asmap_path = args.GetDataDirNet() / asmap_path; - } - if (!fs::exists(asmap_path)) { - InitError(strprintf(_("Could not find asmap file %s"), fs::quoted(fs::PathToString(asmap_path)))); - return false; - } - std::vector asmap{DecodeAsmap(asmap_path)}; - if (asmap.size() == 0) { - InitError(strprintf(_("Could not parse asmap file %s"), fs::quoted(fs::PathToString(asmap_path)))); - return false; - } - const uint256 asmap_version = AsmapVersion(asmap); - node.netgroupman = std::make_unique(NetGroupManager::WithLoadedAsmap(std::move(asmap))); LogInfo("Using asmap version %s for IP bucketing", asmap_version.ToString()); } else { node.netgroupman = std::make_unique(NetGroupManager::NoAsmap()); diff --git a/src/node/data/ip_asn.dat b/src/node/data/ip_asn.dat new file mode 100644 index 00000000000..9fad606c3b1 Binary files /dev/null and b/src/node/data/ip_asn.dat differ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9d948118600..eac3ec5942f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -25,6 +25,7 @@ function(create_test_config) set_configure_variable(BUILD_DAEMON BUILD_BITCOIND) set_configure_variable(BUILD_FUZZ_BINARY ENABLE_FUZZ_BINARY) set_configure_variable(WITH_ZMQ ENABLE_ZMQ) + set_configure_variable(WITH_EMBEDDED_ASMAP ENABLE_EMBEDDED_ASMAP) set_configure_variable(ENABLE_EXTERNAL_SIGNER ENABLE_EXTERNAL_SIGNER) set_configure_variable(WITH_USDT ENABLE_USDT_TRACEPOINTS) set_configure_variable(ENABLE_IPC ENABLE_IPC) diff --git a/test/config.ini.in b/test/config.ini.in index 40b9395b01c..20fa36b94ef 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -25,6 +25,7 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_BINARY_TRUE@ENABLE_FUZZ_BINARY=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true +@ENABLE_EMBEDDED_ASMAP_TRUE@ENABLE_EMBEDDED_ASMAP=true @ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true @ENABLE_USDT_TRACEPOINTS_TRUE@ENABLE_USDT_TRACEPOINTS=true @ENABLE_IPC_TRUE@ENABLE_IPC=true diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 8ad59c82ccf..310788f2cbf 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -66,12 +66,19 @@ class AsmapTest(BitcoinTestFramework): self.start_node(0, [f'-asmap={name}']) os.remove(filename) - def test_unspecified_asmap(self): - msg = "Error: -asmap requires a file path. Use -asmap=." - for arg in ['-asmap', '-asmap=']: - self.log.info(f'Test bitcoind {arg} (and no filename specified)') - self.stop_node(0) - self.node.assert_start_raises_init_error(extra_args=[arg], expected_msg=msg) + def test_embedded_asmap(self): + if self.is_embedded_asmap_compiled(): + self.log.info('Test bitcoind -asmap (using embedded map data)') + for arg in ['-asmap', '-asmap=1']: + self.stop_node(0) + with self.node.assert_debug_log(["Opened asmap data", "from embedded byte array"]): + self.start_node(0, [arg]) + else: + self.log.info('Test bitcoind -asmap (compiled without embedded map data)') + for arg in ['-asmap', '-asmap=1']: + self.stop_node(0) + msg = "Error: Embedded asmap data not available" + self.node.assert_start_raises_init_error(extra_args=[arg], expected_msg=msg) def test_asmap_interaction_with_addrman_containing_entries(self): self.log.info("Test bitcoind -asmap restart with addrman containing new and tried entries") @@ -127,7 +134,7 @@ class AsmapTest(BitcoinTestFramework): self.test_noasmap_arg() self.test_asmap_with_absolute_path() self.test_asmap_with_relative_path() - self.test_unspecified_asmap() + self.test_embedded_asmap() self.test_asmap_interaction_with_addrman_containing_entries() self.test_asmap_with_missing_file() self.test_empty_asmap() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 7970da52595..67def9b5668 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1019,6 +1019,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Checks whether the zmq module was compiled.""" return self.config["components"].getboolean("ENABLE_ZMQ") + def is_embedded_asmap_compiled(self): + """Checks whether ASMap data was embedded during compilation.""" + return self.config["components"].getboolean("ENABLE_EMBEDDED_ASMAP") + def is_usdt_compiled(self): """Checks whether the USDT tracepoints were compiled.""" return self.config["components"].getboolean("ENABLE_USDT_TRACEPOINTS")