diff --git a/.github/ci-windows-cross.py b/.github/ci-windows-cross.py index 90cd59c7fe7..e41ec4adce5 100755 --- a/.github/ci-windows-cross.py +++ b/.github/ci-windows-cross.py @@ -10,6 +10,9 @@ import subprocess import sys from pathlib import Path +sys.path.append(str(Path(__file__).resolve().parent.parent / "test")) +from download_utils import download_script_assets + def run(cmd, **kwargs): print("+ " + shlex.join(cmd), flush=True) @@ -81,6 +84,9 @@ def prepare_tests(): run(cmd_download_prev_rel) run([sys.executable, "-m", "pip", "install", "pyzmq"]) + dest = workspace / "unit_test_data" + download_script_assets(dest) + def run_functional_tests(): workspace = Path.cwd() @@ -117,6 +123,8 @@ def run_functional_tests(): def run_unit_tests(): + workspace = Path.cwd() + os.environ["DIR_UNIT_TEST_DATA"] = str(workspace / "unit_test_data") # Can't use ctest here like other jobs as we don't have a CMake build tree. commands = [ ["./bin/test_bitcoin-qt.exe"], diff --git a/.github/ci-windows.py b/.github/ci-windows.py index 16d7db7a2ed..20c27a8fd94 100755 --- a/.github/ci-windows.py +++ b/.github/ci-windows.py @@ -10,6 +10,9 @@ import subprocess import sys from pathlib import Path +sys.path.append(str(Path(__file__).resolve().parent.parent / "test")) +from download_utils import download_script_assets + def run(cmd, **kwargs): print("+ " + shlex.join(cmd), flush=True) @@ -126,10 +129,13 @@ def check_manifests(ci_type): def prepare_tests(ci_type): + workspace = Path.cwd() if ci_type == "standard": run([sys.executable, "-m", "pip", "install", "pyzmq"]) + dest = workspace / "unit_test_data" + download_script_assets(dest) elif ci_type == "fuzz": - repo_dir = str(Path.cwd() / "qa-assets") + repo_dir = str(workspace / "qa-assets") clone_cmd = [ "git", "clone", @@ -143,11 +149,13 @@ def prepare_tests(ci_type): def run_tests(ci_type): - build_dir = Path.cwd() / "build" + workspace = Path.cwd() + build_dir = workspace / "build" num_procs = str(os.process_cpu_count()) release_bin = build_dir / "bin" / "Release" if ci_type == "standard": + os.environ["DIR_UNIT_TEST_DATA"] = str(workspace / "unit_test_data") test_envs = { "BITCOIN_BIN": "bitcoin.exe", "BITCOIND": "bitcoind.exe", @@ -180,7 +188,7 @@ def run_tests(ci_type): "--jobs", num_procs, "--quiet", - f"--tmpdirprefix={Path.cwd()}", + f"--tmpdirprefix={workspace}", "--combinedlogslen=99999999", *shlex.split(os.environ.get("TEST_RUNNER_EXTRA", "").strip()), ] @@ -195,7 +203,7 @@ def run_tests(ci_type): num_procs, "--loglevel", "DEBUG", - str(Path.cwd() / "qa-assets" / "fuzz_corpora"), + str(workspace / "qa-assets" / "fuzz_corpora"), ] run(fuzz_cmd) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fedde674e2..70305f26186 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -428,13 +428,12 @@ jobs: - name: Check executable manifests run: py -3 .github/ci-windows-cross.py check_manifests + - name: Prepare Windows test environment + run: py -3 .github/ci-windows-cross.py prepare_tests + - name: Run unit tests run: py -3 .github/ci-windows-cross.py run_unit_tests - - name: Prepare Windows test environment - run: | - py -3 .github/ci-windows-cross.py prepare_tests - - name: Run functional tests env: TEST_RUNNER_EXTRA: "--timeout-factor=${{ env.TEST_RUNNER_TIMEOUT_FACTOR }} ${{ case(github.event_name == 'pull_request', '', '--extended') }}" diff --git a/test/download_utils.py b/test/download_utils.py new file mode 100644 index 00000000000..c7fd6cf0c6f --- /dev/null +++ b/test/download_utils.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +import sys +import time +import urllib.request + + +def download_from_url(url, archive): + print(f"Fetching: {url}") + last_print_time = time.time() + + def progress_hook(progress_bytes, total_size): + nonlocal last_print_time + now = time.time() + percent = min(100, (progress_bytes * 100) / total_size) + bar_length = 40 + filled_length = int(bar_length * percent / 100) + bar = '#' * filled_length + '-' * (bar_length - filled_length) + if now - last_print_time >= 1 or percent >= 100: + print(f'\rDownloading: [{bar}] {percent:.1f}%', flush=True, end="") + last_print_time = now + + with urllib.request.urlopen(url) as response: + if response.status != 200: + raise RuntimeError(f"HTTP request failed with status code: {response.status}") + + sock_info = response.fp.raw._sock.getpeername() + print(f"Connected to {sock_info[0]}") + + total_size = int(response.getheader("Content-Length")) + progress_bytes = 0 + + with open(archive, 'wb') as file: + while True: + chunk = response.read(8192) + if not chunk: + break + file.write(chunk) + progress_bytes += len(chunk) + progress_hook(progress_bytes, total_size) + + if progress_bytes < total_size: + raise RuntimeError(f"Download incomplete: expected {total_size} bytes, got {progress_bytes} bytes") + + print('\n', flush=True, end="") # Flush to avoid error output on the same line. + + +def download_script_assets(script_assets_dir): + script_assets_dir.mkdir(parents=True, exist_ok=True) + script_assets = script_assets_dir / "script_assets_test.json" + url = "https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json" + try: + download_from_url(url, script_assets) + except Exception as e: + print(f"\nDownload failed: {e}", file=sys.stderr) + print("Retrying download after failure ...", file=sys.stderr) + time.sleep(12) + try: + download_from_url(url, script_assets) + except Exception as e2: + sys.exit(e2) diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 4984c1b23e3..cd77c55e461 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -3,10 +3,6 @@ # Copyright (c) 2018-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Download or build previous releases. -# Needs curl and tar to download a release, or the build dependencies when -# building a release. import argparse import contextlib @@ -20,9 +16,11 @@ import shutil import subprocess import sys import time -import urllib.request import zipfile +sys.path.append(str(Path(__file__).resolve().parent)) +from download_utils import download_from_url + TAR = os.getenv('TAR', 'tar') SHA256_SUMS = { @@ -106,45 +104,6 @@ def pushd(new_dir) -> None: os.chdir(previous_dir) -def download_from_url(url, archive): - last_print_time = time.time() - - def progress_hook(progress_bytes, total_size): - nonlocal last_print_time - now = time.time() - percent = min(100, (progress_bytes * 100) / total_size) - bar_length = 40 - filled_length = int(bar_length * percent / 100) - bar = '#' * filled_length + '-' * (bar_length - filled_length) - if now - last_print_time >= 1 or percent >= 100: - print(f'\rDownloading: [{bar}] {percent:.1f}%', flush=True, end="") - last_print_time = now - - with urllib.request.urlopen(url) as response: - if response.status != 200: - raise RuntimeError(f"HTTP request failed with status code: {response.status}") - - sock_info = response.fp.raw._sock.getpeername() - print(f"Connected to {sock_info[0]}") - - total_size = int(response.getheader("Content-Length")) - progress_bytes = 0 - - with open(archive, 'wb') as file: - while True: - chunk = response.read(8192) - if not chunk: - break - file.write(chunk) - progress_bytes += len(chunk) - progress_hook(progress_bytes, total_size) - - if progress_bytes < total_size: - raise RuntimeError(f"Download incomplete: expected {total_size} bytes, got {progress_bytes} bytes") - - print('\n', flush=True, end="") # Flush to avoid error output on the same line. - - def download_binary(tag, args) -> int: if Path(tag).is_dir(): if not args.remove_dir: @@ -170,8 +129,6 @@ def download_binary(tag, args) -> int: archive = f'bitcoin-{tag[1:]}-{host}.{archive_format}' archive_url = f'https://bitcoincore.org/{bin_path}/{archive}' - print(f'Fetching: {archive_url}') - try: download_from_url(archive_url, archive) except Exception as e: