From da0732ebd5ee5e4cb95c192519c9bf0c9e5757c6 Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Mon, 24 Feb 2025 16:04:30 -0500 Subject: [PATCH 1/3] ci: containerize Continuous Integration jobs This frees us from GitHub actions brownouts and early deprecation by no longer using Microsoft-maintained images in favor of bare images from Docker Hub. The latter keeps images available for a longer time, regardless of whether or not Canonical directly supports them. Benefits: - Decouples our release cycle from what Microsoft is willing to spend support effort on. - Reduces bloat in base images Because the bare ubuntu images contain a lot less pre-installed tooling, some changes in orchestration need to be made for this to be successful, as well as some workarounds can be removed: - All GitHub Actions base operating systems are set to ubuntu-24.04, as all we use from this now is containerd. - All docker images remain ubuntu:20.04 for now, as this is needed for symbol compatibility (and for now, we use the same version in Gitian builds) - Until the base image used for GitHub Actions and the provided containerd is providing sandboxed mounting on /proc, every process that needs custom binfmts needs to be ran privileged. Currently this means all windows builds need this. see: https://lore.kernel.org/all/20211028103114.2849140-2-brauner@kernel.org/ - Staying on custom binfmts, the Microsoft-maintained image contained mono runtimes and reduced compatibility with wine, for which we had workarounds in place. These are no longer needed and have been removed. - For future troubleshooting, the currently installed binfmts and the kernel version are verbosely displayed inside jobs - Because GitHub Actions does not allow us to pass zero arguments to container options, whenever there are none a dummy environment value 1DOGE is set to 1DOGE. - For macOS, the libbz2-dev library was missing from our system dependencies, this was masked by it being installed by default in the Microsoft-maintained image. - Bare containers do not come configured with a timezone, so this needed to be added to container initialization before any apt calls are made. - Because Microsoft runs the container under uid 1001, which does not exist in the bare ubuntu:20.04 image, a special cleanup step for the qa/cache needed to be made, or else the build caches would fail to upload at the end of the CI run. --- .github/workflows/ci.yml | 88 +++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1a8f72b5..e95514eec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,7 @@ jobs: PYTHON_DEBUG: "1" CACHE_NONCE: "1" WINEDEBUG: fixme-all + WINEPREFIX: /tmp/wineprefix/ SDK_URL: https://depends.dogecoincore.org BIONIC_LIEF_WHL: lief-0.12.3-cp36-cp36m-linux_x86_64.whl @@ -44,7 +45,7 @@ jobs: include: - name: i686-linux host: i686-pc-linux-gnu - os: ubuntu-20.04 + container: ubuntu:20.04 packages: g++-multilib bc python3-zmq run-bench: true test-script: | @@ -58,7 +59,7 @@ jobs: goal: install - name: armhf-linux host: arm-linux-gnueabihf - os: ubuntu-20.04 + container: ubuntu:20.04 packages: g++-arm-linux-gnueabihf qemu-user-static qemu-user run-bench: false test-script: | @@ -70,7 +71,7 @@ jobs: goal: install - name: aarch64-linux-experimental host: aarch64-linux-gnu - os: ubuntu-20.04 + container: ubuntu:20.04 packages: g++-aarch64-linux-gnu qemu-user-static qemu-user run-bench: false test-script: | @@ -82,7 +83,7 @@ jobs: goal: install - name: aarch64-linux host: aarch64-linux-gnu - os: ubuntu-20.04 + container: ubuntu:20.04 packages: g++-aarch64-linux-gnu qemu-user-static qemu-user run-bench: false test-script: | @@ -94,7 +95,7 @@ jobs: goal: install - name: x86_64-linux-nowallet host: x86_64-unknown-linux-gnu - os: ubuntu-20.04 + container: ubuntu:20.04 packages: python3 run-bench: true test-script: | @@ -106,7 +107,7 @@ jobs: goal: install - name: x86_64-linux-dbg host: x86_64-unknown-linux-gnu - os: ubuntu-20.04 + container: ubuntu:20.04 packages: bc python3-zmq run-bench: true test-script: | @@ -120,22 +121,17 @@ jobs: goal: install - name: i686-win host: i686-w64-mingw32 - os: ubuntu-20.04 - packages: python3 nsis g++-mingw-w64-i686 wine-stable winehq-stable bc wine-binfmt binfmt-support + container: ubuntu:20.04 + container-options: --privileged + packages: python3 nsis g++-mingw-w64-i686 wine32 bc wine-binfmt binfmt-support preinstall: | - sudo dpkg --add-architecture i386 - OS_FLAVOR=$(cat /etc/*ease | grep UBUNTU_CODENAME | cut -d "=" -f 2) - if [ ! -d "/etc/apt/keyrings" ]; then sudo mkdir -pm755 /etc/apt/keyrings; fi - sudo wget -O /etc/apt/keyrings/winehq-archive.key https://dl.winehq.org/wine-builds/winehq.key - sudo wget -NP /etc/apt/sources.list.d/ https://dl.winehq.org/wine-builds/ubuntu/dists/$OS_FLAVOR/winehq-$OS_FLAVOR.sources - sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - sudo apt-get update - sudo apt-get install -y --allow-downgrades libgd3/$OS_FLAVOR libpcre2-8-0/$OS_FLAVOR libpcre2-16-0/$OS_FLAVOR libpcre2-32-0/$OS_FLAVOR libpcre2-posix2/$OS_FLAVOR - sudo apt-get purge -yq libmono* moby* mono* php* libgdiplus libpcre2-posix3 libzip4 + dpkg --add-architecture i386 postinstall: | - sudo update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix - sudo update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix - sudo update-binfmts --import /usr/share/binfmts/wine + update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix + update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix + update-binfmts --import /usr/share/binfmts/wine + update-binfmts --enable + update-binfmts --display run-bench: false test-script: | make check $MAKEJOBS VERBOSE=1 @@ -146,13 +142,15 @@ jobs: goal: install - name: x86_64-win host: x86_64-w64-mingw32 - os: ubuntu-20.04 + container: ubuntu:20.04 + container-options: --privileged packages: python3 nsis g++-mingw-w64-x86-64 wine64 bc wine-binfmt postinstall: | - sudo dpkg -s mono-runtime && sudo apt-get remove mono-runtime || echo "Very nothing to uninstall." - sudo update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix - sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix - sudo update-binfmts --import /usr/share/binfmts/wine + update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix + update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix + update-binfmts --import /usr/share/binfmts/wine + update-binfmts --enable + update-binfmts --display run-bench: false test-script: | make check $MAKEJOBS VERBOSE=1 @@ -163,13 +161,15 @@ jobs: goal: install - name: x86_64-win-experimental host: x86_64-w64-mingw32 - os: ubuntu-20.04 + container: ubuntu:20.04 + container-options: --privileged packages: python3 nsis g++-mingw-w64-x86-64 wine64 bc wine-binfmt postinstall: | - sudo dpkg -s mono-runtime && sudo apt-get remove mono-runtime || echo "Very nothing to uninstall." - sudo update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix - sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix - sudo update-binfmts --import /usr/share/binfmts/wine + update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix + update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix + update-binfmts --import /usr/share/binfmts/wine + update-binfmts --enable + update-binfmts --display run-bench: false test-script: | make check $MAKEJOBS VERBOSE=1 @@ -180,8 +180,8 @@ jobs: goal: install - name: x86_64-macos host: x86_64-apple-darwin11 - os: ubuntu-20.04 - packages: cmake imagemagick libcap-dev librsvg2-bin libz-dev libtiff-tools libtinfo5 xorriso + container: ubuntu:20.04 + packages: cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 xorriso run-bench: false check-security: true check-symbols: false @@ -192,7 +192,7 @@ jobs: sdk-shasum: "bec9d089ebf2e2dd59b1a811a38ec78ebd5da18cbbcd6ab39d1e59f64ac5033f" - name: x86_64-linux-experimental host: x86_64-linux-gnu - os: ubuntu-20.04 + container: ubuntu:20.04 packages: bc python3-zmq run-bench: true test-script: | @@ -203,20 +203,30 @@ jobs: config-opts: "--enable-experimental --enable-scrypt-sse2 --with-intel-avx2 --with-gui=qt5 --enable-zmq --enable-glibc-back-compat --enable-reduce-exports --enable-c++14" goal: install - runs-on: ${{ matrix.os }} + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.container }} + options: ${{ matrix.container-options == '' && '-e 1DOGE=1DOGE' || matrix.container-options }} steps: + - name: Configure container + run: | + ln -snf /usr/share/zoneinfo/Etc/UTC /etc/localtime + echo Etc/UTC > /etc/timezone + mkdir -p $WINEPREFIX + uname -a + - name: Pre install if: ${{ matrix.preinstall }} run: ${{ matrix.preinstall }} - name: Install packages run: | - sudo apt-get update - sudo apt-get install build-essential libtool autotools-dev automake \ + apt-get update + apt-get install -y build-essential libtool autotools-dev automake \ pkg-config bsdmainutils curl ca-certificates ccache rsync git \ procps bison python3 python3-pip python3-setuptools python3-wheel - sudo apt-get install ${{ matrix.packages }} + apt-get install -y ${{ matrix.packages }} python3 -m pip install setuptools==70.3.0 --upgrade - name: Install custom lief wheel @@ -300,6 +310,10 @@ jobs: if: ${{ matrix.check-symbols }} run: make -C src check-symbols + - name: Cleanup qa artifacts + run: | + rm -rf qa/cache || true + - name: Upload artifacts uses: actions/upload-artifact@v4 with: From af0362a496e2718f5f9d52ffe8a2d97187054ed7 Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Tue, 25 Feb 2025 16:43:35 -0500 Subject: [PATCH 2/3] ci: remove fallback version of lief for ubuntu:bionic --- .github/workflows/ci.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e95514eec..744785636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,6 @@ jobs: WINEDEBUG: fixme-all WINEPREFIX: /tmp/wineprefix/ SDK_URL: https://depends.dogecoincore.org - BIONIC_LIEF_WHL: lief-0.12.3-cp36-cp36m-linux_x86_64.whl strategy: fail-fast: false @@ -228,14 +227,7 @@ jobs: procps bison python3 python3-pip python3-setuptools python3-wheel apt-get install -y ${{ matrix.packages }} python3 -m pip install setuptools==70.3.0 --upgrade - - - name: Install custom lief wheel - if: matrix.os == 'ubuntu-18.04' - run: python3 -m pip install $SDK_URL/$BIONIC_LIEF_WHL - - - name: Install standard lief wheel - if: matrix.os != 'ubuntu-18.04' - run: python3 -m pip install lief + python3 -m pip install lief - name: Post install if: ${{ matrix.postinstall }} From 5136b94f947ae2378c1e2ac244e2de4e0795e501 Mon Sep 17 00:00:00 2001 From: Patrick Lodder Date: Tue, 25 Feb 2025 16:44:34 -0500 Subject: [PATCH 3/3] ci: move linter to ubuntu:24.04 - do not containerize the linter (for now) as all it does is a python3 check on translations. --- .github/workflows/linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index e077f0d98..81ae2520b 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -10,7 +10,7 @@ on: jobs: translations: name: Check Translations - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Lint translation files