From 8c7f0056291d3f145633c453170272d951b128f5 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 26 Nov 2024 22:53:46 -0500 Subject: [PATCH 1/6] test: add is_ipc_compiled() and skip_if_no_ipc() functions Expose ENABLE_IPC build option to functional tests so new tests can test IPC-only features. --- test/CMakeLists.txt | 1 + test/config.ini.in | 1 + test/functional/test_framework/test_framework.py | 9 +++++++++ 3 files changed, 11 insertions(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0b6520a0810..e2075346006 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -26,6 +26,7 @@ function(create_test_config) set_configure_variable(WITH_ZMQ ENABLE_ZMQ) set_configure_variable(ENABLE_EXTERNAL_SIGNER ENABLE_EXTERNAL_SIGNER) set_configure_variable(WITH_USDT ENABLE_USDT_TRACEPOINTS) + set_configure_variable(ENABLE_IPC ENABLE_IPC) configure_file(config.ini.in config.ini USE_SOURCE_PERMISSIONS @ONLY) endfunction() diff --git a/test/config.ini.in b/test/config.ini.in index e66791eb6f0..f695dec47f2 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -26,3 +26,4 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true @ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true @ENABLE_USDT_TRACEPOINTS_TRUE@ENABLE_USDT_TRACEPOINTS=true +@ENABLE_IPC_TRUE@ENABLE_IPC=true diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 639421d3aea..abe62bc2be1 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1016,6 +1016,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_cli_compiled(): raise SkipTest("bitcoin-cli has not been compiled.") + def skip_if_no_ipc(self): + """Skip the running test if ipc is not compiled.""" + if not self.is_ipc_compiled(): + raise SkipTest("ipc has not been compiled.") + def skip_if_no_previous_releases(self): """Skip the running test if previous releases are not available.""" if not self.has_previous_releases(): @@ -1075,6 +1080,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Checks whether the USDT tracepoints were compiled.""" return self.config["components"].getboolean("ENABLE_USDT_TRACEPOINTS") + def is_ipc_compiled(self): + """Checks whether ipc was compiled.""" + return self.config["components"].getboolean("ENABLE_IPC") + def has_blockfile(self, node, filenum: str): return (node.blocks_path/ f"blk{filenum}.dat").is_file() From 3cceb60a7153f5512ad292c2315740af64fbc41d Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 26 Nov 2024 22:53:46 -0500 Subject: [PATCH 2/6] test: Provide path to `bitcoin` binary Set new `BitcoinTestFramework.binary_paths.bitcoin_bin` property with path to the `bitcoin` wrapper binary. This allows new tests for `bitcoin-mine` in #30437 and `bitcoin-cli` in #32297 to find the `bitcoin` binary and call `bitcoin -m` to start nodes with IPC support. This way the new tests can run whenever the ENABLE_IPC build option is enabled, instead of only running when the `BITCOIN_CMD` environment variable is set to `bitcoin -m` --- .github/workflows/ci.yml | 1 + test/functional/test_framework/test_framework.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f729a62dcf..87dd21075b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -275,6 +275,7 @@ jobs: if: matrix.job-type == 'standard' working-directory: build env: + BITCOIN_BIN: '${{ github.workspace }}\build\bin\Release\bitcoin.exe' BITCOIND: '${{ github.workspace }}\build\bin\Release\bitcoind.exe' BITCOINCLI: '${{ github.workspace }}\build\bin\Release\bitcoin-cli.exe' BITCOINTX: '${{ github.workspace }}\build\bin\Release\bitcoin-tx.exe' diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index abe62bc2be1..bc722ab7e8a 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -278,6 +278,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): paths = types.SimpleNamespace() binaries = { + "bitcoin": "BITCOIN_BIN", "bitcoind": "BITCOIND", "bitcoin-cli": "BITCOINCLI", "bitcoin-util": "BITCOINUTIL", @@ -285,6 +286,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): "bitcoin-chainstate": "BITCOINCHAINSTATE", "bitcoin-wallet": "BITCOINWALLET", } + # Set paths to bitcoin core binaries allowing overrides with environment + # variables. for binary, env_variable_name in binaries.items(): default_filename = os.path.join( self.config["environment"]["BUILDDIR"], From 3cc9a06c8dd5a8ae58a52c85f40dafbf74bbf740 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Mon, 18 Aug 2025 11:19:22 -0400 Subject: [PATCH 3/6] test: Add TestNode ipcbind option With this change, tests can specify `self.extra_init = [{ipcbind: True}]` to start a node listening on an IPC socket, instead of needing to choose which node binary to invoke and what `self.extra_args=[["-ipcbind=..."]]` value to pass to it. The eliminates boilerplate code #30437 (interface_ipc_mining.py), #32297 (interface_ipc_cli.py), and #33201 (interface_ipc.py) previously needed in their test setup. --- .../test_framework/test_framework.py | 33 ++++++++++++------- test/functional/test_framework/test_node.py | 27 +++++++++++++-- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index bc722ab7e8a..03c00bec7ac 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -76,9 +76,9 @@ class Binaries: self.paths = paths self.bin_dir = bin_dir - def node_argv(self): + def node_argv(self, **kwargs): "Return argv array that should be used to invoke bitcoind" - return self._argv("node", self.paths.bitcoind) + return self._argv("node", self.paths.bitcoind, **kwargs) def rpc_argv(self): "Return argv array that should be used to invoke bitcoin-cli" @@ -101,16 +101,19 @@ class Binaries: "Return argv array that should be used to invoke bitcoin-chainstate" return self._argv("chainstate", self.paths.bitcoinchainstate) - def _argv(self, command, bin_path): + def _argv(self, command, bin_path, need_ipc=False): """Return argv array that should be used to invoke the command. It - either uses the bitcoin wrapper executable (if BITCOIN_CMD is set), or - the direct binary path (bitcoind, etc). When bin_dir is set (by tests - calling binaries from previous releases) it always uses the direct - path.""" + either uses the bitcoin wrapper executable (if BITCOIN_CMD is set or + need_ipc is True), or the direct binary path (bitcoind, etc). When + bin_dir is set (by tests calling binaries from previous releases) it + always uses the direct path.""" if self.bin_dir is not None: return [os.path.join(self.bin_dir, os.path.basename(bin_path))] - elif self.paths.bitcoin_cmd is not None: - return self.paths.bitcoin_cmd + [command] + elif self.paths.bitcoin_cmd is not None or need_ipc: + # If the current test needs IPC functionality, use the bitcoin + # wrapper binary and append -m so it calls multiprocess binaries. + bitcoin_cmd = self.paths.bitcoin_cmd or [self.paths.bitcoin_bin] + return bitcoin_cmd + (["-m"] if need_ipc else []) + [command] else: return [bin_path] @@ -158,6 +161,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.noban_tx_relay: bool = False self.nodes: list[TestNode] = [] self.extra_args = None + self.extra_init = None self.network_thread = None self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond self.supports_cli = True @@ -553,15 +557,15 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): bin_dirs.append(bin_dir) + extra_init = [{}] * num_nodes if self.extra_init is None else self.extra_init # type: ignore[var-annotated] + assert_equal(len(extra_init), num_nodes) assert_equal(len(extra_confs), num_nodes) assert_equal(len(extra_args), num_nodes) assert_equal(len(versions), num_nodes) assert_equal(len(bin_dirs), num_nodes) for i in range(num_nodes): args = list(extra_args[i]) - test_node_i = TestNode( - i, - get_datadir_path(self.options.tmpdir, i), + init = dict( chain=self.chain, rpchost=rpchost, timewait=self.rpc_timeout, @@ -578,6 +582,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): v2transport=self.options.v2transport, uses_wallet=self.uses_wallet, ) + init.update(extra_init[i]) + test_node_i = TestNode( + i, + get_datadir_path(self.options.tmpdir, i), + **init) self.nodes.append(test_node_i) if not test_node_i.version_is_at_least(170000): # adjust conf for pre 17 diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 3b0814635aa..66536627823 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -11,6 +11,7 @@ from enum import Enum import json import logging import os +import pathlib import platform import re import subprocess @@ -19,6 +20,7 @@ import time import urllib.parse import collections import shlex +import shutil import sys from pathlib import Path @@ -52,6 +54,13 @@ CLI_MAX_ARG_SIZE = 131071 # many systems have a 128kb limit per arg (MAX_ARG_STR NULL_BLK_XOR_KEY = bytes([0] * NUM_XOR_BYTES) BITCOIN_PID_FILENAME_DEFAULT = "bitcoind.pid" +if sys.platform.startswith("linux"): + UNIX_PATH_MAX = 108 # includes the trailing NUL +elif sys.platform.startswith(("darwin", "freebsd", "netbsd", "openbsd")): + UNIX_PATH_MAX = 104 +else: # safest portable value + UNIX_PATH_MAX = 92 + class FailedToStartError(Exception): """Raised when a node fails to start correctly.""" @@ -77,7 +86,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir_path, *, chain, rpchost, timewait, timeout_factor, binaries, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, v2transport=False, uses_wallet=False): + def __init__(self, i, datadir_path, *, chain, rpchost, timewait, timeout_factor, binaries, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, v2transport=False, uses_wallet=False, ipcbind=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -109,7 +118,7 @@ class TestNode(): # Configuration for logging is set as command-line args rather than in the bitcoin.conf file. # This means that starting a bitcoind using the temp dir to debug a failed test won't # spam debug.log. - self.args = self.binaries.node_argv() + [ + self.args = self.binaries.node_argv(need_ipc=ipcbind) + [ f"-datadir={self.datadir_path}", "-logtimemicros", "-debug", @@ -121,6 +130,17 @@ class TestNode(): if uses_wallet is not None and not uses_wallet: self.args.append("-disablewallet") + self.ipc_tmp_dir = None + if ipcbind: + self.ipc_socket_path = self.chain_path / "node.sock" + if len(os.fsencode(self.ipc_socket_path)) < UNIX_PATH_MAX: + self.args.append("-ipcbind=unix") + else: + # Work around default CI path exceeding maximum socket path length. + self.ipc_tmp_dir = pathlib.Path(tempfile.mkdtemp(prefix="test-ipc-")) + self.ipc_socket_path = self.ipc_tmp_dir / "node.sock" + self.args.append(f"-ipcbind=unix:{self.ipc_socket_path}") + # Use valgrind, expect for previous release binaries if use_valgrind and version is None: default_suppressions_file = Path(__file__).parents[3] / "contrib" / "valgrind.supp" @@ -208,6 +228,9 @@ class TestNode(): # this destructor is called. print(self._node_msg("Cleaning up leftover process"), file=sys.stderr) self.process.kill() + if self.ipc_tmp_dir: + print(self._node_msg(f"Cleaning up ipc directory {str(self.ipc_tmp_dir)!r}")) + shutil.rmtree(self.ipc_tmp_dir) def __getattr__(self, name): """Dispatches any unrecognised messages to the RPC connection or a CLI instance.""" From 8d2ee88fa2a569410d09e5e1a4f6c92ff9d5e5d1 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 21 Aug 2025 03:16:07 -0400 Subject: [PATCH 4/6] tests: add functional tests for IPC interface Co-Authored-By: ismaelsadeeq Co-Authored-By: ryanofsky Co-Authored-By: TheCharlatan Co-Authored-By: Sjors Provoost --- test/README.md | 16 ++ test/functional/interface_ipc.py | 188 ++++++++++++++++++ .../test_framework/test_framework.py | 7 + test/functional/test_runner.py | 1 + 4 files changed, 212 insertions(+) create mode 100755 test/functional/interface_ipc.py diff --git a/test/README.md b/test/README.md index 37f2c072178..f44e1679d9f 100644 --- a/test/README.md +++ b/test/README.md @@ -34,6 +34,22 @@ The ZMQ functional test requires a python ZMQ library. To install it: - on Unix, run `sudo apt-get install python3-zmq` - on mac OS, run `pip3 install pyzmq` +The IPC functional test requires a python IPC library. `pip3 install pycapnp` may work, but if not, install it from source: + +```sh +git clone -b v2.1.0 https://github.com/capnproto/pycapnp +pip3 install ./pycapnp +``` + +If that does not work, try adding `-C force-bundled-libcapnp=True` to the `pip` command. +Depending on the system, it may be necessary to install and run in a venv: + +```sh +python -m venv venv +git clone -b v2.1.0 https://github.com/capnproto/pycapnp +venv/bin/pip3 install ./pycapnp -C force-bundled-libcapnp=True +venv/bin/python3 build/test/functional/interface_ipc.py +``` On Windows the `PYTHONUTF8` environment variable must be set to 1: diff --git a/test/functional/interface_ipc.py b/test/functional/interface_ipc.py new file mode 100755 index 00000000000..8410fa24bef --- /dev/null +++ b/test/functional/interface_ipc.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the IPC (multiprocess) interface.""" +import asyncio +from io import BytesIO +from pathlib import Path +import shutil +from test_framework.messages import (CBlock, CTransaction, ser_uint256) +from test_framework.test_framework import (BitcoinTestFramework, assert_equal) +from test_framework.wallet import MiniWallet + +# Test may be skipped and not have capnp installed +try: + import capnp # type: ignore[import] # noqa: F401 +except ImportError: + pass + + +class IPCInterfaceTest(BitcoinTestFramework): + + def skip_test_if_missing_module(self): + self.skip_if_no_ipc() + self.skip_if_no_py_capnp() + + def load_capnp_modules(self): + if capnp_bin := shutil.which("capnp"): + # Add the system cap'nproto path so include/capnp/c++.capnp can be found. + capnp_dir = Path(capnp_bin).parent.parent / "include" + else: + # If there is no system cap'nproto, the pycapnp module should have its own "bundled" + # includes at this location. If pycapnp was installed with bundled capnp, + # capnp/c++.capnp can be found here. + capnp_dir = Path(capnp.__path__[0]).parent + src_dir = Path(self.config['environment']['SRCDIR']) / "src" + mp_dir = src_dir / "ipc" / "libmultiprocess" / "include" + imports = [str(capnp_dir), str(src_dir), str(mp_dir)] + return { + "proxy": capnp.load(str(mp_dir / "mp" / "proxy.capnp"), imports=imports), + "init": capnp.load(str(src_dir / "ipc" / "capnp" / "init.capnp"), imports=imports), + "echo": capnp.load(str(src_dir / "ipc" / "capnp" / "echo.capnp"), imports=imports), + "mining": capnp.load(str(src_dir / "ipc" / "capnp" / "mining.capnp"), imports=imports), + } + + def set_test_params(self): + self.num_nodes = 1 + + def setup_nodes(self): + self.extra_init = [{"ipcbind": True}] + super().setup_nodes() + # Use this function to also load the capnp modules (we cannot use set_test_params for this, + # as it is being called before knowing whether capnp is available). + self.capnp_modules = self.load_capnp_modules() + + async def make_capnp_init_ctx(self): + node = self.nodes[0] + # Establish a connection, and create Init proxy object. + connection = await capnp.AsyncIoStream.create_unix_connection(node.ipc_socket_path) + client = capnp.TwoPartyClient(connection) + init = client.bootstrap().cast_as(self.capnp_modules['init'].Init) + # Create a remote thread on the server for the IPC calls to be executed in. + threadmap = init.construct().threadMap + thread = threadmap.makeThread("pythread").result + ctx = self.capnp_modules['proxy'].Context() + ctx.thread = thread + # Return both. + return ctx, init + + async def parse_and_deserialize_block(self, block_template, ctx): + block_data = BytesIO((await block_template.result.getBlock(ctx)).result) + block = CBlock() + block.deserialize(block_data) + return block + + def run_echo_test(self): + self.log.info("Running echo test") + async def async_routine(): + ctx, init = await self.make_capnp_init_ctx() + self.log.debug("Create Echo proxy object") + echo = init.makeEcho(ctx).result + self.log.debug("Test a few invocations of echo") + for s in ["hallo", "", "haha"]: + result_eval = (await echo.echo(ctx, s)).result + assert_equal(s, result_eval) + self.log.debug("Destroy the Echo object") + echo.destroy(ctx) + asyncio.run(capnp.run(async_routine())) + + def run_mining_test(self): + self.log.info("Running mining test") + block_hash_size = 32 + block_header_size = 80 + timeout = 1000.0 # 1000 milliseconds + miniwallet = MiniWallet(self.nodes[0]) + + async def async_routine(): + ctx, init = await self.make_capnp_init_ctx() + self.log.debug("Create Mining proxy object") + mining = init.makeMining(ctx) + self.log.debug("Test simple inspectors") + assert (await mining.result.isTestChain(ctx)) + assert (await mining.result.isInitialBlockDownload(ctx)) + blockref = await mining.result.getTip(ctx) + assert blockref.hasResult + assert_equal(len(blockref.result.hash), block_hash_size) + current_block_height = self.nodes[0].getchaintips()[0]["height"] + assert blockref.result.height == current_block_height + self.log.debug("Mine a block") + wait = mining.result.waitTipChanged(ctx, blockref.result.hash, ) + self.generate(self.nodes[0], 1) + newblockref = await wait + assert_equal(len(newblockref.result.hash), block_hash_size) + assert_equal(newblockref.result.height, current_block_height + 1) + self.log.debug("Wait for timeout") + wait = mining.result.waitTipChanged(ctx, newblockref.result.hash, timeout) + oldblockref = await wait + assert_equal(len(newblockref.result.hash), block_hash_size) + assert_equal(oldblockref.result.hash, newblockref.result.hash) + assert_equal(oldblockref.result.height, newblockref.result.height) + + self.log.debug("Create a template") + opts = self.capnp_modules['mining'].BlockCreateOptions() + opts.useMempool = True + opts.blockReservedWeight = 4000 + opts.coinbaseOutputMaxAdditionalSigops = 0 + template = mining.result.createNewBlock(opts) + self.log.debug("Test some inspectors of Template") + header = await template.result.getBlockHeader(ctx) + assert_equal(len(header.result), block_header_size) + block = await self.parse_and_deserialize_block(template, ctx) + assert_equal(ser_uint256(block.hashPrevBlock), newblockref.result.hash) + assert len(block.vtx) >= 1 + txfees = await template.result.getTxFees(ctx) + assert_equal(len(txfees.result), 0) + txsigops = await template.result.getTxSigops(ctx) + assert_equal(len(txsigops.result), 0) + coinbase_data = BytesIO((await template.result.getCoinbaseTx(ctx)).result) + coinbase = CTransaction() + coinbase.deserialize(coinbase_data) + assert_equal(coinbase.vin[0].prevout.hash, 0) + self.log.debug("Wait for a new template") + waitoptions = self.capnp_modules['mining'].BlockWaitOptions() + waitoptions.timeout = timeout + waitnext = template.result.waitNext(ctx, waitoptions) + self.generate(self.nodes[0], 1) + template2 = await waitnext + block2 = await self.parse_and_deserialize_block(template2, ctx) + assert_equal(len(block2.vtx), 1) + self.log.debug("Wait for another, but time out") + template3 = await template2.result.waitNext(ctx, waitoptions) + assert_equal(template3.to_dict(), {}) + self.log.debug("Wait for another, get one after increase in fees in the mempool") + waitnext = template2.result.waitNext(ctx, waitoptions) + miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]) + template4 = await waitnext + block3 = await self.parse_and_deserialize_block(template4, ctx) + assert_equal(len(block3.vtx), 2) + self.log.debug("Wait again, this should return the same template, since the fee threshold is zero") + template5 = await template4.result.waitNext(ctx, waitoptions) + block4 = await self.parse_and_deserialize_block(template5, ctx) + assert_equal(len(block4.vtx), 2) + waitoptions.feeThreshold = 1 + self.log.debug("Wait for another, get one after increase in fees in the mempool") + waitnext = template5.result.waitNext(ctx, waitoptions) + miniwallet.send_self_transfer(fee_rate=10, from_node=self.nodes[0]) + template6 = await waitnext + block4 = await self.parse_and_deserialize_block(template6, ctx) + assert_equal(len(block4.vtx), 3) + self.log.debug("Wait for another, but time out, since the fee threshold is set now") + template7 = await template6.result.waitNext(ctx, waitoptions) + assert_equal(template7.to_dict(), {}) + self.log.debug("Destroy template objects") + template.result.destroy(ctx) + template2.result.destroy(ctx) + template3.result.destroy(ctx) + template4.result.destroy(ctx) + template5.result.destroy(ctx) + template6.result.destroy(ctx) + template7.result.destroy(ctx) + asyncio.run(capnp.run(async_routine())) + + def run_test(self): + self.run_echo_test() + self.run_mining_test() + +if __name__ == '__main__': + IPCInterfaceTest(__file__).main() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 03c00bec7ac..0b9295cef53 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -964,6 +964,13 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): except ImportError: raise SkipTest("sqlite3 module not available.") + def skip_if_no_py_capnp(self): + """Attempt to import the capnp package and skip the test if the import fails.""" + try: + import capnp # type: ignore[import] # noqa: F401 + except ImportError: + raise SkipTest("capnp module not available.") + def skip_if_no_python_bcc(self): """Attempt to import the bcc package and skip the tests if the import fails.""" try: diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index f2e514a7a46..31c9805e8eb 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -339,6 +339,7 @@ BASE_SCRIPTS = [ 'rpc_scantxoutset.py', 'feature_unsupported_utxo_db.py', 'feature_logging.py', + 'interface_ipc.py', 'feature_anchors.py', 'mempool_datacarrier.py', 'feature_coinstatsindex.py', From 6aee573bfcf6bce9b08b7e96d8e4608a457d5add Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 21 Aug 2025 03:16:23 -0400 Subject: [PATCH 5/6] ci: enable IPC tests in CI --- .github/workflows/ci.yml | 6 ++++++ ci/test/00_setup_env_native_asan.sh | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87dd21075b1..85fdd7eccbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,7 @@ jobs: - run: | sudo apt-get update sudo apt-get install clang mold ccache build-essential cmake ninja-build pkgconf python3-zmq libevent-dev libboost-dev libsqlite3-dev systemtap-sdt-dev libzmq3-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev capnproto libcapnp-dev -y + sudo pip3 install --break-system-packages pycapnp - name: Compile and run tests run: | # Run tests on commits after the last merge commit and before the PR head commit @@ -152,6 +153,11 @@ jobs: brew install --quiet python@3 || brew link --overwrite python@3 brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent zeromq qt@6 qrencode capnp + - name: Install Python packages + run: | + git clone -b v2.1.0 https://github.com/capnproto/pycapnp + pip3 install ./pycapnp -C force-bundled-libcapnp=True --break-system-packages + - name: Set Ccache directory run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index a6f7385448e..e785f6ab51f 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -20,7 +20,8 @@ fi export CONTAINER_NAME=ci_native_asan export APT_LLVM_V="21" -export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libevent-dev libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto" +export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libevent-dev libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto python3-pip" +export PIP_PACKAGES="--break-system-packages pycapnp" export NO_DEPENDS=1 export GOAL="install" export CI_LIMIT_STACK_SIZE=1 From a341e11ac92b30aac856049c696a9ac39854569d Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Thu, 21 Aug 2025 15:08:45 +0200 Subject: [PATCH 6/6] ci: test IPC on additional hosts Install pycapnp on all (active) CI hosts which have IPC enabled and run the functional tests. Except for previous_releases, which uses an older version of pip that doesn't support --break-system-packages. --- ci/test/00_setup_env_native_centos.sh | 2 +- ci/test/00_setup_env_native_msan.sh | 3 ++- ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh | 3 ++- ci/test/00_setup_env_native_tsan.sh | 3 ++- ci/test/00_setup_env_native_valgrind.sh | 3 ++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/test/00_setup_env_native_centos.sh b/ci/test/00_setup_env_native_centos.sh index ef77b2f0c8c..998ddaf45f8 100755 --- a/ci/test/00_setup_env_native_centos.sh +++ b/ci/test/00_setup_env_native_centos.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_centos export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream10" export CI_BASE_PACKAGES="gcc-c++ glibc-devel libstdc++-devel ccache make ninja-build git python3 python3-pip which patch xz procps-ng rsync coreutils bison e2fsprogs cmake dash" -export PIP_PACKAGES="pyzmq" +export PIP_PACKAGES="pyzmq pycapnp" export DEP_OPTS="DEBUG=1" export GOAL="install" export BITCOIN_CONFIG="\ diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 17ac78b4423..4ff82614e9d 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -14,7 +14,8 @@ LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${L export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev" +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev python3-pip" +export PIP_PACKAGES="--break-system-packages pycapnp" export DEP_OPTS="DEBUG=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export CI_LIMIT_STACK_SIZE=1 diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 74bc891aafc..58561e55204 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -9,7 +9,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:bookworm" # Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md -export PACKAGES="python3-zmq clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" +export PACKAGES="python3-zmq python3-pip clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" +export PIP_PACKAGES="--break-system-packages pycapnp" export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_LIB=ON -DBUILD_SHARED_LIBS=ON" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 3871c37fd5a..bf4b68412b2 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -11,7 +11,8 @@ export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" export APT_LLVM_V="21" LIBCXX_DIR="/cxx_build/" LIBCXX_FLAGS="-fsanitize=thread -nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" -export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev python3-zmq" +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev python3-zmq python3-pip" +export PIP_PACKAGES="--break-system-packages pycapnp" export DEP_OPTS="CC=clang CXX=clang++ CXXFLAGS='${LIBCXX_FLAGS}' NO_QT=1" export GOAL="install" export CI_LIMIT_STACK_SIZE=1 diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 99dbe807f5b..d6c45755df2 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -8,7 +8,8 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind python3-zmq libevent-dev libboost-dev libzmq3-dev libsqlite3-dev libcapnp-dev capnproto" +export PACKAGES="valgrind python3-zmq libevent-dev libboost-dev libzmq3-dev libsqlite3-dev libcapnp-dev capnproto python3-pip" +export PIP_PACKAGES="--break-system-packages pycapnp" export USE_VALGRIND=1 export NO_DEPENDS=1 # bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547