diff --git a/.github/ci-windows.py b/.github/ci-windows.py new file mode 100755 index 00000000000..cbb5b27f242 --- /dev/null +++ b/.github/ci-windows.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +import argparse +import os +import shlex +import subprocess +import sys +from pathlib import Path + + +def run(cmd, **kwargs): + print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) + try: + return subprocess.run(cmd, **kwargs) + except Exception as e: + sys.exit(str(e)) + + +GENERATE_OPTIONS = { + "standard": [ + "-DBUILD_BENCH=ON", + "-DBUILD_KERNEL_LIB=ON", + "-DBUILD_UTIL_CHAINSTATE=ON", + "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON", + ], + "fuzz": [ + "-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON", + "-DVCPKG_MANIFEST_FEATURES=wallet", + "-DBUILD_GUI=OFF", + "-DWITH_ZMQ=OFF", + "-DBUILD_FOR_FUZZING=ON", + "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON", + ], +} + + +def generate(ci_type): + command = [ + "cmake", + "-B", + "build", + "-Werror=dev", + "--preset", + "vs2022", + ] + GENERATE_OPTIONS[ci_type] + run(command) + + +def build(): + command = [ + "cmake", + "--build", + "build", + "--config", + "Release", + ] + if run(command + ["-j", str(os.process_cpu_count())], check=False).returncode != 0: + print("Build failure. Verbose build follows.") + run(command + ["-j1", "--verbose"]) + + +def check_manifests(ci_type): + if ci_type != "standard": + print(f"Skipping manifest validation for '{ci_type}' ci type.") + return + + release_dir = Path.cwd() / "build" / "bin" / "Release" + manifest_path = release_dir / "bitcoind.manifest" + cmd_bitcoind_manifest = [ + "mt.exe", + "-nologo", + f"-inputresource:{release_dir / 'bitcoind.exe'}", + f"-out:{manifest_path}", + ] + run(cmd_bitcoind_manifest) + print(manifest_path.read_text()) + + skips = { # Skip as they currently do not have manifests + "fuzz.exe", + "bench_bitcoin.exe", + "test_bitcoin-qt.exe", + "test_kernel.exe", + "bitcoin-chainstate.exe", + } + for entry in release_dir.iterdir(): + if entry.suffix.lower() != ".exe": + continue + if entry.name in skips: + print(f"Skipping {entry.name} (no manifest present)") + continue + print(f"Checking {entry.name}") + cmd_check_manifest = [ + "mt.exe", + "-nologo", + f"-inputresource:{entry}", + "-validate_manifest", + ] + run(cmd_check_manifest) + + +def prepare_tests(ci_type): + if ci_type == "standard": + run([sys.executable, "-m", "pip", "install", "pyzmq"]) + elif ci_type == "fuzz": + repo_dir = os.path.join(os.getcwd(), "qa-assets") + clone_cmd = [ + "git", + "clone", + "--depth=1", + "https://github.com/bitcoin-core/qa-assets", + repo_dir, + ] + run(clone_cmd) + print("Using qa-assets repo from commit ...") + run(["git", "-C", repo_dir, "log", "-1"]) + + +def run_tests(ci_type): + build_dir = "build" + num_procs = str(os.process_cpu_count()) + release_bin = os.path.join(os.getcwd(), build_dir, "bin", "Release") + + if ci_type == "standard": + test_envs = { + "BITCOIN_BIN": "bitcoin.exe", + "BITCOIND": "bitcoind.exe", + "BITCOINCLI": "bitcoin-cli.exe", + "BITCOIN_BENCH": "bench_bitcoin.exe", + "BITCOINTX": "bitcoin-tx.exe", + "BITCOINUTIL": "bitcoin-util.exe", + "BITCOINWALLET": "bitcoin-wallet.exe", + "BITCOINCHAINSTATE": "bitcoin-chainstate.exe", + } + for var, exe in test_envs.items(): + os.environ[var] = os.path.join(release_bin, exe) + + ctest_cmd = [ + "ctest", + "--test-dir", + build_dir, + "--output-on-failure", + "--stop-on-failure", + "-j", + num_procs, + "--build-config", + "Release", + ] + run(ctest_cmd) + + test_cmd = [ + sys.executable, + os.path.join(build_dir, "test", "functional", "test_runner.py"), + "--jobs", + num_procs, + "--quiet", + f"--tmpdirprefix={os.getcwd()}", + "--combinedlogslen=99999999", + *shlex.split(os.environ.get("TEST_RUNNER_EXTRA", "").strip()), + ] + run(test_cmd) + + elif ci_type == "fuzz": + os.environ["BITCOINFUZZ"] = os.path.join(release_bin, "fuzz.exe") + fuzz_cmd = [ + sys.executable, + os.path.join(build_dir, "test", "fuzz", "test_runner.py"), + "--par", + num_procs, + "--loglevel", + "DEBUG", + os.path.join(os.getcwd(), "qa-assets", "fuzz_corpora"), + ] + run(fuzz_cmd) + + +def main(): + parser = argparse.ArgumentParser(description="Utility to run Windows CI steps.") + parser.add_argument("ci_type", choices=GENERATE_OPTIONS, help="CI type to run.") + steps = [ + "generate", + "build", + "check_manifests", + "prepare_tests", + "run_tests", + ] + parser.add_argument("step", choices=steps, help="CI step to perform.") + args = parser.parse_args() + + if args.step == "generate": + generate(args.ci_type) + elif args.step == "build": + build() + elif args.step == "check_manifests": + check_manifests(args.ci_type) + elif args.step == "prepare_tests": + prepare_tests(args.ci_type) + elif args.step == "run_tests": + run_tests(args.ci_type) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4482c1ed403..7f5299cfbd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ defaults: jobs: runners: name: '[meta] determine runners' - runs-on: ubuntu-latest + runs-on: ubuntu-slim outputs: provider: ${{ steps.runners.outputs.provider }} steps: @@ -218,10 +218,8 @@ jobs: job-type: [standard, fuzz] include: - job-type: standard - generate-options: '-DBUILD_BENCH=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON' job-name: 'Windows native, VS 2022' - job-type: fuzz - generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="wallet" -DBUILD_GUI=OFF -DWITH_ZMQ=OFF -DBUILD_FOR_FUZZING=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON' job-name: 'Windows native, fuzz, VS 2022' steps: @@ -257,6 +255,10 @@ jobs: # Workaround for libevent, which requires CMake 3.1 but is incompatible with CMake >= 4.0. sed -i '1s/^/set(ENV{CMAKE_POLICY_VERSION_MINIMUM} 3.5)\n/' "${VCPKG_INSTALLATION_ROOT}/scripts/ports.cmake" + - name: Set VCPKG_ROOT + run: | + echo "VCPKG_ROOT=${VCPKG_INSTALLATION_ROOT}" >> "$GITHUB_ENV" + - name: vcpkg tools cache uses: actions/cache@v5 with: @@ -272,7 +274,7 @@ jobs: - name: Generate build system run: | - cmake -B build -Werror=dev --preset vs2022 -DCMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" ${{ matrix.generate-options }} + py -3 .github/ci-windows.py ${{ matrix.job-type }} generate - name: Save vcpkg binary cache uses: actions/cache/save@v4 @@ -282,75 +284,28 @@ jobs: key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} - name: Build - working-directory: build run: | - cmake --build . -j $NUMBER_OF_PROCESSORS --config Release + py -3 .github/ci-windows.py ${{ matrix.job-type }} build - name: Check executable manifests - if: matrix.job-type == 'standard' - working-directory: build - shell: pwsh -Command "$PSVersionTable; $PSNativeCommandUseErrorActionPreference = $true; $ErrorActionPreference = 'Stop'; & '{0}'" run: | - mt.exe -nologo -inputresource:bin\Release\bitcoind.exe -out:bitcoind.manifest - Get-Content bitcoind.manifest + py -3 .github/ci-windows.py ${{ matrix.job-type }} check_manifests - Get-ChildItem -Filter "bin\Release\*.exe" | ForEach-Object { - $exeName = $_.Name - - # Skip as they currently do not have manifests - if ($exeName -eq "fuzz.exe" -or $exeName -eq "bench_bitcoin.exe" -or $exeName -eq "test_bitcoin-qt.exe" -or $exeName -eq "test_kernel.exe" -or $exeName -eq "bitcoin-chainstate.exe") { - Write-Host "Skipping $exeName (no manifest present)" - return - } - - Write-Host "Checking $exeName" - & mt.exe -nologo -inputresource:$_.FullName -validate_manifest - } - - - name: Run test suite - if: matrix.job-type == 'standard' - working-directory: build + - name: Prepare tests run: | - ctest --output-on-failure --stop-on-failure -j $NUMBER_OF_PROCESSORS -C Release + py -3 .github/ci-windows.py ${{ matrix.job-type }} prepare_tests - - name: Run functional tests - if: matrix.job-type == 'standard' - working-directory: build + - name: Run tests env: - BITCOIN_BIN: '${{ github.workspace }}\build\bin\Release\bitcoin.exe' - BITCOIND: '${{ github.workspace }}\build\bin\Release\bitcoind.exe' - BITCOINCLI: '${{ github.workspace }}\build\bin\Release\bitcoin-cli.exe' - BITCOIN_BENCH: '${{ github.workspace }}\build\bin\Release\bench_bitcoin.exe' - BITCOINTX: '${{ github.workspace }}\build\bin\Release\bitcoin-tx.exe' - BITCOINUTIL: '${{ github.workspace }}\build\bin\Release\bitcoin-util.exe' - BITCOINWALLET: '${{ github.workspace }}\build\bin\Release\bitcoin-wallet.exe' - BITCOINCHAINSTATE: '${{ github.workspace }}\build\bin\Release\bitcoin-chainstate.exe' - TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} + TEST_RUNNER_EXTRA: "--timeout-factor=${{ env.TEST_RUNNER_TIMEOUT_FACTOR }} ${{ case(github.event_name == 'pull_request', '', '--extended') }}" run: | - py -3 -m pip install pyzmq - py -3 test/functional/test_runner.py --jobs $NUMBER_OF_PROCESSORS --quiet --tmpdirprefix="${RUNNER_TEMP}" --combinedlogslen=99999999 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} - - - name: Clone corpora - if: matrix.job-type == 'fuzz' - run: | - git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${RUNNER_TEMP}/qa-assets" - cd "${RUNNER_TEMP}/qa-assets" - echo "Using qa-assets repo from commit ..." - git log -1 - - - name: Run fuzz tests - if: matrix.job-type == 'fuzz' - working-directory: build - env: - BITCOINFUZZ: '${{ github.workspace }}\build\bin\Release\fuzz.exe' - run: | - py -3 test/fuzz/test_runner.py --par $NUMBER_OF_PROCESSORS --loglevel DEBUG "${RUNNER_TEMP}/qa-assets/fuzz_corpora" + py -3 .github/ci-windows.py ${{ matrix.job-type }} run_tests record-frozen-commit: # Record frozen commit, so that the native tests on cross-builds can run on # the exact same commit id of the build. name: '[meta] record frozen commit' - runs-on: ubuntu-latest + runs-on: ubuntu-slim outputs: commit: ${{ steps.record-commit.outputs.commit }} steps: