Merge bitcoin/bitcoin#31560: rpc: allow writing UTXO set to a named pipe

b19caeea098f92a7f72aaeee49573358f4b153a3 doc: add release note for #31560 (named pipe support for `dumptxoutset` RPC) (Sebastian Falbesoner)
61a5460d0d6cd174d395c51333def798fe7442fe test: add test for utxo-to-sqlite conversion using named pipe (Sebastian Falbesoner)
2e8072edbeb20a8c05c0dbd06ca105bc4dd07b96 rpc: support writing UTXO set dump (`dumptxoutset`) to a named pipe (Sebastian Falbesoner)

Pull request description:

  This PR slightly modifies the `dumptxoutset` RPC to allow writing the UTXO set dump into a [named pipe](https://askubuntu.com/a/449192), so that the output data can be consumed by another process, see #31373. Taking use of this with the utxo-to-sqlite.py tool (introduced in #27432), creating an UTXO set in SQLite3 format is possible on the fly. E.g. for signet:
  ```
  $ mkfifo /tmp/utxo_fifo && ./build/bin/bitcoin-cli -signet dumptxoutset /tmp/utxo_fifo latest &
  $ ./contrib/utxo-tools/utxo_to_sqlite.py /tmp/utxo_fifo ./utxo.sqlite
  UTXO Snapshot for Signet at block hash 000000012711f0a4e741be4a22792982..., contains 61848352 coins
  1048576 coins converted [1.70%], 2.800s passed since start
  ....
  ....
  60817408 coins converted [98.33%], 159.598s passed since start
  {
    "coins_written": 61848352,
    "base_hash": "000000012711f0a4e741be4a22792982370f51326db20fca955c7d45da97f768",
    "base_height": 294305,
    "path": "/tmp/utxo_fifo",
    "txoutset_hash": "34ae7fe7af33f58d4b83e00ecfc3b9605d927f154e7a94401226922f8e3f534e",
    "nchaintx": 28760852
  }
  TOTAL: 61848352 coins written to ./utxo.sqlite, snapshot height is 294305.
  ```
  Note that the `dumptxoutset` RPC calculates an UTXO set hash as a first step before any data is emitted, so especially on mainnet it takes quite a while until the conversion starts and something is happening visibly.

ACKs for top commit:
  ajtowns:
    utACK b19caeea098f92a7f72aaeee49573358f4b153a3
  sedited:
    Re-ACK b19caeea098f92a7f72aaeee49573358f4b153a3

Tree-SHA512: 7101563d0dba15439cdef8c8fb535f8593d5a779ff04208e2d72382a3f99072db8eac3651d1b3fe72c5e1f03e164efb281c3030d45d0723b943ebbbcf2a841d6

doc/release-notes-31560.md

@@ -0,0 +1,8 @@
+Updated RPCs
+------------
+
+- The `dumptxoutset` RPC now supports writing to a named pipe
+  on UNIX-like systems (see mkfifo(1) and mkfifo(3) man pages).
+  This allows the raw UTXO set data to be consumed directly by
+  another process (e.g., the `contrib/utxo-tools/utxo_to_sqlite.py`
+  conversion script) without first writing it to disk. (#31560)

src/rpc/blockchain.cpp

@@ -3109,11 +3109,12 @@ static RPCHelpMan dumptxoutset()
 
     const ArgsManager& args{EnsureAnyArgsman(request.context)};
     const fs::path path = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(self.Arg<std::string_view>("path")));
+    const auto path_info{fs::status(path)};
     // Write to a temporary path and then move into `path` on completion
-    // to avoid confusion due to an interruption.
-    const fs::path temppath = path + ".incomplete";
+    // to avoid confusion due to an interruption. If a named pipe passed, write directly to it.
+    const fs::path temppath = fs::is_fifo(path_info) ? path : path + ".incomplete";
 
-    if (fs::exists(path)) {
+    if (fs::exists(path_info) && !fs::is_fifo(path_info)) {
         throw JSONRPCError(
             RPC_INVALID_PARAMETER,
             path.utf8string() + " already exists. If you are sure this is what you want, "
@@ -3197,7 +3198,9 @@ static RPCHelpMan dumptxoutset()
                                         path,
                                         temppath,
                                         node.rpc_interruption_point);
-    fs::rename(temppath, path);
+    if (!fs::is_fifo(path_info)) {
+        fs::rename(temppath, path);
+    }
 
     result.pushKV("path", path.utf8string());
     return result;

src/util/fs.h

@@ -96,6 +96,10 @@ static inline bool exists(const path& p)
 {
     return std::filesystem::exists(p);
 }
+static inline bool exists(const std::filesystem::file_status& s)
+{
+    return std::filesystem::exists(s);
+}
 
 // Allow explicit quoted stream I/O.
 static inline auto quoted(const std::string& s)

test/functional/tool_utxo_to_sqlite.py

@@ -4,7 +4,8 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test utxo-to-sqlite conversion tool"""
 from itertools import product
-import os.path
+import os
+import platform
 try:
     import sqlite3
 except ImportError:
@@ -135,6 +136,19 @@ class UtxoToSqliteTest(BitcoinTestFramework):
             assert_equal(muhash_sqlite, muhash_compact_serialized)
             self.log.info('')
 
+        if platform.system() != "Windows":  # FIFOs are not available on Windows
+            self.log.info('Convert UTXO set directly (without intermediate dump) via named pipe')
+            fifo_filename = os.path.join(self.options.tmpdir, "utxos.fifo")
+            os.mkfifo(fifo_filename)
+            output_direct_filename = os.path.join(self.options.tmpdir, "utxos_direct.sqlite")
+            p = subprocess.Popen([sys.executable, utxo_to_sqlite_path, fifo_filename, output_direct_filename],
+                                 stderr=subprocess.STDOUT)
+            node.dumptxoutset(fifo_filename, "latest")
+            p.wait(timeout=10)
+            muhash_direct_sqlite = calculate_muhash_from_sqlite_utxos(output_direct_filename, "hex", "hex")
+            assert_equal(muhash_sqlite, muhash_direct_sqlite)
+            os.remove(fifo_filename)
+
 
 if __name__ == "__main__":
     UtxoToSqliteTest(__file__).main()