mirror of
https://github.com/bitcoin/bitcoin.git
synced 2026-03-02 01:36:13 +00:00
Merge bitcoin/bitcoin#34500: ci: Print verbose Windows CI build failure
fa90277d22e1e6662e827e662eb6bda344cdcb20 ci: Use ubuntu-slim for [meta] runners (MarcoFalke)
fa9627af9f89b2e2c04ae564d11d1af148c9682b ci: Rely on cmake --preset toolchain file (MarcoFalke)
fa3f89acaa7ae3332f4bf13aa91fcff44902990c ci: Add check_manifests to ci-windows.py (MarcoFalke)
1111079a16b92bb34ee2f8d4ae3529fbcb3180b2 ci: Add run_tests step to ci-windows.py (MarcoFalke)
fa561682ce4052bed91fac11f8fc872af5132437 ci: [refactor] Add .github/ci-windows.py prepare_tests step (MarcoFalke)
fa3e607c6dfb220684a65638b35ac5a4d3b1fcb7 ci: Print verbose Windows CI build failure (MarcoFalke)
4444808dd3a704ec0f5d97e2c736cc917656c491 ci: [refactor] Add .github/ci-windows.py build step (MarcoFalke)
fabdd4e82342b57e8e7750c5292e9fe0d598105b ci: Refactor Windows CI into script (MarcoFalke)
Pull request description:
Just like all the other CI configs, the Windows one should print a single and readable build failure at the end.
Also, includes a bunch of Windows CI refactors.
ACKs for top commit:
m3dwards:
ACK fa90277d22e1e6662e827e662eb6bda344cdcb20
hebasto:
ACK fa90277d22e1e6662e827e662eb6bda344cdcb20.
willcl-ark:
utACK fa90277d22e1e6662e827e662eb6bda344cdcb20
Tree-SHA512: 00443289e3d8b6d60d1347934d9d4b16098e8c36b6325467e5804b1869714201c4f7e932da3be44392c73e4713a1f52cd8af481894d36c6a281ba7238d43434e
This commit is contained in:
Hennadii Stepanov
commit
64294c8909
206
.github/ci-windows.py
vendored
Executable file
206
.github/ci-windows.py
vendored
Executable file
@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or https://opensource.org/license/mit/.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run(cmd, **kwargs):
|
||||
print("+ " + shlex.join(cmd), flush=True)
|
||||
kwargs.setdefault("check", True)
|
||||
try:
|
||||
return subprocess.run(cmd, **kwargs)
|
||||
except Exception as e:
|
||||
sys.exit(str(e))
|
||||
|
||||
|
||||
GENERATE_OPTIONS = {
|
||||
"standard": [
|
||||
"-DBUILD_BENCH=ON",
|
||||
"-DBUILD_KERNEL_LIB=ON",
|
||||
"-DBUILD_UTIL_CHAINSTATE=ON",
|
||||
"-DCMAKE_COMPILE_WARNING_AS_ERROR=ON",
|
||||
],
|
||||
"fuzz": [
|
||||
"-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON",
|
||||
"-DVCPKG_MANIFEST_FEATURES=wallet",
|
||||
"-DBUILD_GUI=OFF",
|
||||
"-DWITH_ZMQ=OFF",
|
||||
"-DBUILD_FOR_FUZZING=ON",
|
||||
"-DCMAKE_COMPILE_WARNING_AS_ERROR=ON",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def generate(ci_type):
|
||||
command = [
|
||||
"cmake",
|
||||
"-B",
|
||||
"build",
|
||||
"-Werror=dev",
|
||||
"--preset",
|
||||
"vs2022",
|
||||
] + GENERATE_OPTIONS[ci_type]
|
||||
run(command)
|
||||
|
||||
|
||||
def build():
|
||||
command = [
|
||||
"cmake",
|
||||
"--build",
|
||||
"build",
|
||||
"--config",
|
||||
"Release",
|
||||
]
|
||||
if run(command + ["-j", str(os.process_cpu_count())], check=False).returncode != 0:
|
||||
print("Build failure. Verbose build follows.")
|
||||
run(command + ["-j1", "--verbose"])
|
||||
|
||||
|
||||
def check_manifests(ci_type):
|
||||
if ci_type != "standard":
|
||||
print(f"Skipping manifest validation for '{ci_type}' ci type.")
|
||||
return
|
||||
|
||||
release_dir = Path.cwd() / "build" / "bin" / "Release"
|
||||
manifest_path = release_dir / "bitcoind.manifest"
|
||||
cmd_bitcoind_manifest = [
|
||||
"mt.exe",
|
||||
"-nologo",
|
||||
f"-inputresource:{release_dir / 'bitcoind.exe'}",
|
||||
f"-out:{manifest_path}",
|
||||
]
|
||||
run(cmd_bitcoind_manifest)
|
||||
print(manifest_path.read_text())
|
||||
|
||||
skips = { # Skip as they currently do not have manifests
|
||||
"fuzz.exe",
|
||||
"bench_bitcoin.exe",
|
||||
"test_bitcoin-qt.exe",
|
||||
"test_kernel.exe",
|
||||
"bitcoin-chainstate.exe",
|
||||
}
|
||||
for entry in release_dir.iterdir():
|
||||
if entry.suffix.lower() != ".exe":
|
||||
continue
|
||||
if entry.name in skips:
|
||||
print(f"Skipping {entry.name} (no manifest present)")
|
||||
continue
|
||||
print(f"Checking {entry.name}")
|
||||
cmd_check_manifest = [
|
||||
"mt.exe",
|
||||
"-nologo",
|
||||
f"-inputresource:{entry}",
|
||||
"-validate_manifest",
|
||||
]
|
||||
run(cmd_check_manifest)
|
||||
|
||||
|
||||
def prepare_tests(ci_type):
|
||||
if ci_type == "standard":
|
||||
run([sys.executable, "-m", "pip", "install", "pyzmq"])
|
||||
elif ci_type == "fuzz":
|
||||
repo_dir = os.path.join(os.getcwd(), "qa-assets")
|
||||
clone_cmd = [
|
||||
"git",
|
||||
"clone",
|
||||
"--depth=1",
|
||||
"https://github.com/bitcoin-core/qa-assets",
|
||||
repo_dir,
|
||||
]
|
||||
run(clone_cmd)
|
||||
print("Using qa-assets repo from commit ...")
|
||||
run(["git", "-C", repo_dir, "log", "-1"])
|
||||
|
||||
|
||||
def run_tests(ci_type):
|
||||
build_dir = "build"
|
||||
num_procs = str(os.process_cpu_count())
|
||||
release_bin = os.path.join(os.getcwd(), build_dir, "bin", "Release")
|
||||
|
||||
if ci_type == "standard":
|
||||
test_envs = {
|
||||
"BITCOIN_BIN": "bitcoin.exe",
|
||||
"BITCOIND": "bitcoind.exe",
|
||||
"BITCOINCLI": "bitcoin-cli.exe",
|
||||
"BITCOIN_BENCH": "bench_bitcoin.exe",
|
||||
"BITCOINTX": "bitcoin-tx.exe",
|
||||
"BITCOINUTIL": "bitcoin-util.exe",
|
||||
"BITCOINWALLET": "bitcoin-wallet.exe",
|
||||
"BITCOINCHAINSTATE": "bitcoin-chainstate.exe",
|
||||
}
|
||||
for var, exe in test_envs.items():
|
||||
os.environ[var] = os.path.join(release_bin, exe)
|
||||
|
||||
ctest_cmd = [
|
||||
"ctest",
|
||||
"--test-dir",
|
||||
build_dir,
|
||||
"--output-on-failure",
|
||||
"--stop-on-failure",
|
||||
"-j",
|
||||
num_procs,
|
||||
"--build-config",
|
||||
"Release",
|
||||
]
|
||||
run(ctest_cmd)
|
||||
|
||||
test_cmd = [
|
||||
sys.executable,
|
||||
os.path.join(build_dir, "test", "functional", "test_runner.py"),
|
||||
"--jobs",
|
||||
num_procs,
|
||||
"--quiet",
|
||||
f"--tmpdirprefix={os.getcwd()}",
|
||||
"--combinedlogslen=99999999",
|
||||
*shlex.split(os.environ.get("TEST_RUNNER_EXTRA", "").strip()),
|
||||
]
|
||||
run(test_cmd)
|
||||
|
||||
elif ci_type == "fuzz":
|
||||
os.environ["BITCOINFUZZ"] = os.path.join(release_bin, "fuzz.exe")
|
||||
fuzz_cmd = [
|
||||
sys.executable,
|
||||
os.path.join(build_dir, "test", "fuzz", "test_runner.py"),
|
||||
"--par",
|
||||
num_procs,
|
||||
"--loglevel",
|
||||
"DEBUG",
|
||||
os.path.join(os.getcwd(), "qa-assets", "fuzz_corpora"),
|
||||
]
|
||||
run(fuzz_cmd)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Utility to run Windows CI steps.")
|
||||
parser.add_argument("ci_type", choices=GENERATE_OPTIONS, help="CI type to run.")
|
||||
steps = [
|
||||
"generate",
|
||||
"build",
|
||||
"check_manifests",
|
||||
"prepare_tests",
|
||||
"run_tests",
|
||||
]
|
||||
parser.add_argument("step", choices=steps, help="CI step to perform.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.step == "generate":
|
||||
generate(args.ci_type)
|
||||
elif args.step == "build":
|
||||
build()
|
||||
elif args.step == "check_manifests":
|
||||
check_manifests(args.ci_type)
|
||||
elif args.step == "prepare_tests":
|
||||
prepare_tests(args.ci_type)
|
||||
elif args.step == "run_tests":
|
||||
run_tests(args.ci_type)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
73
.github/workflows/ci.yml
vendored
73
.github/workflows/ci.yml
vendored
@ -31,7 +31,7 @@ defaults:
|
||||
jobs:
|
||||
runners:
|
||||
name: '[meta] determine runners'
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-slim
|
||||
outputs:
|
||||
provider: ${{ steps.runners.outputs.provider }}
|
||||
steps:
|
||||
@ -218,10 +218,8 @@ jobs:
|
||||
job-type: [standard, fuzz]
|
||||
include:
|
||||
- job-type: standard
|
||||
generate-options: '-DBUILD_BENCH=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON'
|
||||
job-name: 'Windows native, VS 2022'
|
||||
- job-type: fuzz
|
||||
generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="wallet" -DBUILD_GUI=OFF -DWITH_ZMQ=OFF -DBUILD_FOR_FUZZING=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON'
|
||||
job-name: 'Windows native, fuzz, VS 2022'
|
||||
|
||||
steps:
|
||||
@ -257,6 +255,10 @@ jobs:
|
||||
# Workaround for libevent, which requires CMake 3.1 but is incompatible with CMake >= 4.0.
|
||||
sed -i '1s/^/set(ENV{CMAKE_POLICY_VERSION_MINIMUM} 3.5)\n/' "${VCPKG_INSTALLATION_ROOT}/scripts/ports.cmake"
|
||||
|
||||
- name: Set VCPKG_ROOT
|
||||
run: |
|
||||
echo "VCPKG_ROOT=${VCPKG_INSTALLATION_ROOT}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: vcpkg tools cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
@ -272,7 +274,7 @@ jobs:
|
||||
|
||||
- name: Generate build system
|
||||
run: |
|
||||
cmake -B build -Werror=dev --preset vs2022 -DCMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" ${{ matrix.generate-options }}
|
||||
py -3 .github/ci-windows.py ${{ matrix.job-type }} generate
|
||||
|
||||
- name: Save vcpkg binary cache
|
||||
uses: actions/cache/save@v4
|
||||
@ -282,75 +284,28 @@ jobs:
|
||||
key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }}
|
||||
|
||||
- name: Build
|
||||
working-directory: build
|
||||
run: |
|
||||
cmake --build . -j $NUMBER_OF_PROCESSORS --config Release
|
||||
py -3 .github/ci-windows.py ${{ matrix.job-type }} build
|
||||
|
||||
- name: Check executable manifests
|
||||
if: matrix.job-type == 'standard'
|
||||
working-directory: build
|
||||
shell: pwsh -Command "$PSVersionTable; $PSNativeCommandUseErrorActionPreference = $true; $ErrorActionPreference = 'Stop'; & '{0}'"
|
||||
run: |
|
||||
mt.exe -nologo -inputresource:bin\Release\bitcoind.exe -out:bitcoind.manifest
|
||||
Get-Content bitcoind.manifest
|
||||
py -3 .github/ci-windows.py ${{ matrix.job-type }} check_manifests
|
||||
|
||||
Get-ChildItem -Filter "bin\Release\*.exe" | ForEach-Object {
|
||||
$exeName = $_.Name
|
||||
|
||||
# Skip as they currently do not have manifests
|
||||
if ($exeName -eq "fuzz.exe" -or $exeName -eq "bench_bitcoin.exe" -or $exeName -eq "test_bitcoin-qt.exe" -or $exeName -eq "test_kernel.exe" -or $exeName -eq "bitcoin-chainstate.exe") {
|
||||
Write-Host "Skipping $exeName (no manifest present)"
|
||||
return
|
||||
}
|
||||
|
||||
Write-Host "Checking $exeName"
|
||||
& mt.exe -nologo -inputresource:$_.FullName -validate_manifest
|
||||
}
|
||||
|
||||
- name: Run test suite
|
||||
if: matrix.job-type == 'standard'
|
||||
working-directory: build
|
||||
- name: Prepare tests
|
||||
run: |
|
||||
ctest --output-on-failure --stop-on-failure -j $NUMBER_OF_PROCESSORS -C Release
|
||||
py -3 .github/ci-windows.py ${{ matrix.job-type }} prepare_tests
|
||||
|
||||
- name: Run functional tests
|
||||
if: matrix.job-type == 'standard'
|
||||
working-directory: build
|
||||
- name: Run tests
|
||||
env:
|
||||
BITCOIN_BIN: '${{ github.workspace }}\build\bin\Release\bitcoin.exe'
|
||||
BITCOIND: '${{ github.workspace }}\build\bin\Release\bitcoind.exe'
|
||||
BITCOINCLI: '${{ github.workspace }}\build\bin\Release\bitcoin-cli.exe'
|
||||
BITCOIN_BENCH: '${{ github.workspace }}\build\bin\Release\bench_bitcoin.exe'
|
||||
BITCOINTX: '${{ github.workspace }}\build\bin\Release\bitcoin-tx.exe'
|
||||
BITCOINUTIL: '${{ github.workspace }}\build\bin\Release\bitcoin-util.exe'
|
||||
BITCOINWALLET: '${{ github.workspace }}\build\bin\Release\bitcoin-wallet.exe'
|
||||
BITCOINCHAINSTATE: '${{ github.workspace }}\build\bin\Release\bitcoin-chainstate.exe'
|
||||
TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }}
|
||||
TEST_RUNNER_EXTRA: "--timeout-factor=${{ env.TEST_RUNNER_TIMEOUT_FACTOR }} ${{ case(github.event_name == 'pull_request', '', '--extended') }}"
|
||||
run: |
|
||||
py -3 -m pip install pyzmq
|
||||
py -3 test/functional/test_runner.py --jobs $NUMBER_OF_PROCESSORS --quiet --tmpdirprefix="${RUNNER_TEMP}" --combinedlogslen=99999999 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA}
|
||||
|
||||
- name: Clone corpora
|
||||
if: matrix.job-type == 'fuzz'
|
||||
run: |
|
||||
git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${RUNNER_TEMP}/qa-assets"
|
||||
cd "${RUNNER_TEMP}/qa-assets"
|
||||
echo "Using qa-assets repo from commit ..."
|
||||
git log -1
|
||||
|
||||
- name: Run fuzz tests
|
||||
if: matrix.job-type == 'fuzz'
|
||||
working-directory: build
|
||||
env:
|
||||
BITCOINFUZZ: '${{ github.workspace }}\build\bin\Release\fuzz.exe'
|
||||
run: |
|
||||
py -3 test/fuzz/test_runner.py --par $NUMBER_OF_PROCESSORS --loglevel DEBUG "${RUNNER_TEMP}/qa-assets/fuzz_corpora"
|
||||
py -3 .github/ci-windows.py ${{ matrix.job-type }} run_tests
|
||||
|
||||
record-frozen-commit:
|
||||
# Record frozen commit, so that the native tests on cross-builds can run on
|
||||
# the exact same commit id of the build.
|
||||
name: '[meta] record frozen commit'
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-slim
|
||||
outputs:
|
||||
commit: ${{ steps.record-commit.outputs.commit }}
|
||||
steps:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user