From b3394ab235b93937321ffd08b8924e57855aac38 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 15 Oct 2019 19:28:49 -0400 Subject: [PATCH 1/8] contrib: macdeploy: Correctly generate macOS SDK Previously, we did not include the macOS SDK libc++ headers in our SDK creation process and instead used whichever libc++ headers shipped with the clang package we downloaded in depends. This change adds a script (which works on both GNU/Linux and macOS) to correctly generate the macOS SDK including the libc++ headers. This can be thought of as a simplified rewrite of tpoechtrager's script: https://github.com/tpoechtrager/osxcross/blob/d3392f4eae78f3fa3f1fd065fa423f2712825102/tools/gen_sdk_package.sh The location within the SDK where we place the libc++ headers is chosen such that clang's search path detection logic for sysroots would pick up the headers properly. We also document this change. --- contrib/macdeploy/README.md | 35 ++++++++++---- contrib/macdeploy/gen-sdk | 94 +++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 10 deletions(-) create mode 100755 contrib/macdeploy/gen-sdk diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 68ebb5def..65ac67fde 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -14,13 +14,17 @@ When complete, it will have produced `Bitcoin-Qt.dmg`. ## SDK Extraction -Our current macOS SDK (`macOSX10.14.sdk`) can be extracted from +### Step 1: Obtaining `Xcode.app` + +Our current macOS SDK +(`Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz`) can be +extracted from [Xcode_10.2.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). An Apple ID is needed to download this. -`Xcode.app` is packaged in a `.xip` archive. -This makes the SDK less-trivial to extract on non-macOS machines. -One approach (tested on Debian Buster) is outlined below: +After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` +archive. This makes the SDK less-trivial to extract on non-macOS machines. One +approach (tested on Debian Buster) is outlined below: ```bash @@ -41,17 +45,28 @@ popd xar -xf Xcode_10.2.1.xip -C . ./pbzx/pbzx -n Content | cpio -i - -find Xcode.app -type d -name MacOSX.sdk -exec sh -c 'tar --transform="s/MacOSX.sdk/MacOSX10.14.sdk/" -c -C$(dirname {}) MacOSX.sdk/ | gzip -9n > MacOSX10.14.sdk.tar.gz' \; ``` -on macOS the process is more straightforward: +On macOS the process is more straightforward: ```bash xip -x Xcode_10.2.1.xip -tar -s "/MacOSX.sdk/MacOSX10.14.sdk/" -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk ``` +### Step 2: Generating `Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` + +To generate `Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz`, run +the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the +previous stage) as the first argument. + +```bash +# Generate a Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz from +# the supplied Xcode.app +./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' +``` + +### Historial macOS SDK Extraction Notes + Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from [Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First @@ -91,13 +106,13 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download [Xcode 10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +then download [Xcode_10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). This file is many gigabytes in size, but most (but not all) of what we need is contained only in a single directory: ```bash -Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk +Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk ``` See the SDK Extraction notes above for how to obtain it. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk new file mode 100755 index 000000000..457d8f5e6 --- /dev/null +++ b/contrib/macdeploy/gen-sdk @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import argparse +import plistlib +import pathlib +import sys +import tarfile +import gzip +import os +import contextlib + +@contextlib.contextmanager +def cd(path): + """Context manager that restores PWD even if an exception was raised.""" + old_pwd = os.getcwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(old_pwd) + +def run(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1) + parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False) + + args = parser.parse_args() + + xcode_app = pathlib.Path(args.xcode_app[0]).resolve() + assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app) + + xcode_app_plist = xcode_app.joinpath("Contents/version.plist") + with xcode_app_plist.open('rb') as fp: + pl = plistlib.load(fp) + xcode_version = pl['CFBundleShortVersionString'] + xcode_build_id = pl['ProductBuildVersion'] + print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)) + + sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") + sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist") + with sdk_plist.open('rb') as fp: + pl = plistlib.load(fp) + sdk_version = pl['ProductVersion'] + sdk_build_id = pl['ProductBuildVersion'] + print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id)) + + out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) + + xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1") + assert xcode_libcxx_dir.is_dir() + + if args.out_sdktgz: + out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) + else: + # Construct our own out_sdktgz if not specified on the command line + out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) + + def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): + """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files' + names + + e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking: + + tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir") + + would result in the following members being added to tarfp: + + foo/bar/ -> corresponding to /root/bazdir + foo/bar/qux -> corresponding to /root/bazdir/qux + + """ + def change_tarinfo_base(tarinfo): + if tarinfo.name and tarinfo.name.startswith("./"): + tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) + if tarinfo.linkname and tarinfo.linkname.startswith("./"): + tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + return tarinfo + with cd(dir_to_add): + tarfp.add(".", recursive=True, filter=change_tarinfo_base) + + print("Creating output .tar.gz file...") + with out_sdktgz_path.open("wb") as fp: + with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: + with tarfile.open(mode="w", fileobj=gzf) as tarfp: + print("Adding MacOSX SDK {} files...".format(sdk_version)) + tarfp_add_with_base_change(tarfp, sdk_dir, out_name) + print("Adding libc++ headers...") + tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name)) + print("Done! Find the resulting gzipped tarball at:") + print(out_sdktgz_path.resolve()) + +if __name__ == '__main__': + run() From 3381e4a1892511d4d555853887c89badf4c940a9 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Thu, 18 Jun 2020 18:05:22 -0400 Subject: [PATCH 2/8] Adapt rest of tooling to new SDK naming scheme --- ci/test/00_setup_env_mac.sh | 3 ++- ci/test/05_before_script.sh | 11 +++++++---- contrib/gitian-build.py | 2 +- contrib/gitian-descriptors/gitian-osx.yml | 4 ++-- depends/hosts/darwin.mk | 6 ++++-- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index a4dc54d1c..44eb3d1fe 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -9,7 +9,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross export HOST=x86_64-apple-darwin16 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" -export OSX_SDK=10.14 +export XCODE_VERSION=10.2.1 +export XCODE_BUILD_ID=10E1001 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 368550452..de3388141 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -15,11 +15,14 @@ fi DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources -if [ -n "$OSX_SDK" ] && [ ! -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" +OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" + +if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then + curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" fi -if [ -n "$OSX_SDK" ] && [ -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - DOCKER_EXEC tar -C ${DEPENDS_DIR}/SDKs -xf ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then + DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 4a3df93ce..48e2a9ec5 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -209,7 +209,7 @@ def main(): args.macos = 'm' in args.os # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.14.sdk.tar.gz'): + if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index bbae7201e..756b5fd7c 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -32,7 +32,7 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "MacOSX10.14.sdk.tar.gz" +- "Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz" script: | set -e -o pipefail @@ -90,7 +90,7 @@ script: | BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz # Build dependencies for each host for i in $HOSTS; do diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 82e086a32..272b8230e 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,6 +1,8 @@ OSX_MIN_VERSION=10.12 -OSX_SDK_VERSION=10.14 -OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk +OSX_SDK_VERSION=10.14.4 +XCODE_VERSION=10.2.1 +XCODE_BUILD_ID=10E1001 +OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ From fbcfcf695435c9587e9f9fd2809c4d5350b2558e Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 4 Feb 2020 12:52:11 -0500 Subject: [PATCH 3/8] native_cctools: Don't use libc++ from pinned clang Now that we include the macOS SDK libc++ headers in our macOS SDK tarball, we no longer need this hack to use the libc++ from our pinned clang. --- depends/packages/native_cctools.mk | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 4195230b4..b4957357b 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -72,7 +72,5 @@ define $(package)_stage_cmds cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ - cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ - if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ - if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi + cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil endef From 351beb5c9a67500bcdb9a6ffe15f30e6aca5aa28 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 10 Jun 2020 16:17:51 -0400 Subject: [PATCH 4/8] contrib: macdeploy: Use apple-sdk-tools instead of xar+pbzx --- contrib/macdeploy/README.md | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 65ac67fde..bf4656104 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -27,24 +27,13 @@ archive. This makes the SDK less-trivial to extract on non-macOS machines. One approach (tested on Debian Buster) is outlined below: ```bash +# Install/clone tools needed for extracting Xcode.app +apt install cpio +git clone https://github.com/bitcoin-core/apple-sdk-tools.git -apt install clang cpio git liblzma-dev libxml2-dev libssl-dev make - -git clone https://github.com/tpoechtrager/xar -pushd xar/xar -./configure -make -make install -popd - -git clone https://github.com/NiklasRosenstein/pbzx -pushd pbzx -clang -llzma -lxar pbzx.c -o pbzx -Wl,-rpath=/usr/local/lib -popd - -xar -xf Xcode_10.2.1.xip -C . - -./pbzx/pbzx -n Content | cpio -i +# Unpack Xcode_10.2.1.xip and place the resulting Xcode.app in your current +# working directory +python3 apple-sdk-tools/extract_xcode.py -f Xcode_10.2.1.xip | cpio -d -i ``` On macOS the process is more straightforward: From 85b5e420882b236b81b83acb672b4f5fa4899965 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Thu, 11 Jun 2020 14:48:15 -0400 Subject: [PATCH 5/8] contrib: macdeploy: Remove historical extraction notes --- contrib/macdeploy/README.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index bf4656104..21c0d9200 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -54,21 +54,6 @@ previous stage) as the first argument. ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` -### Historial macOS SDK Extraction Notes - -Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from -[Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). -The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First -ensure the DMG file is in the current directory, and then run the script. You -may wish to delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the -directory) when you've confirmed the extraction succeeded. - -```bash -apt-get install p7zip-full sleuthkit -contrib/macdeploy/extract-osx-sdk.sh -rm -rf 5.hfs MacOSX10.11.sdk -``` - ## Deterministic macOS DMG Notes Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple `binutils` (`ld`, `ar`, etc) and DMG authoring tools. From 5c2c835433a80c204da1335daa51a014670c7324 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 16 Jun 2020 14:17:01 -0400 Subject: [PATCH 6/8] depends: bump MacOS toolchain clang 6.0.1 -> 8.0.0 cctools 921 -> 949.0.1 ld64 409.12 -> 530 --- depends/packages/native_cctools.mk | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index b4957357b..bdebd1186 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,14 +1,14 @@ package=native_cctools -$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f +$(package)_version=4da2f3b485bcf4cef526f30c0b8c0bcda99cdbb4 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04 +$(package)_sha256_hash=a2d491c0981cef72fee2b833598f20f42a6c44a7614a61c439bda93d56446fec $(package)_build_subdir=cctools -$(package)_clang_version=6.0.1 +$(package)_clang_version=8.0.0 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0 +$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0 $(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive @@ -72,5 +72,5 @@ define $(package)_stage_cmds cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ - cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil + cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil endef From 2418f739f75824d6689369f326b960cec254cf56 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 16 Jun 2020 14:20:19 -0400 Subject: [PATCH 7/8] macos: Bump to xcode 11.3.1 and 10.15 SDK This gets us a newer SDK with c++17 support and retains 10.12 back-compat. Co-authored-by: Carl Dong --- ci/test/00_setup_env_mac.sh | 4 ++-- contrib/gitian-build.py | 2 +- contrib/gitian-descriptors/gitian-osx.yml | 4 ++-- contrib/macdeploy/README.md | 18 +++++++++--------- depends/hosts/darwin.mk | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 44eb3d1fe..45a29928c 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -9,8 +9,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross export HOST=x86_64-apple-darwin16 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" -export XCODE_VERSION=10.2.1 -export XCODE_BUILD_ID=10E1001 +export XCODE_VERSION=11.3.1 +export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 48e2a9ec5..d498c9e2c 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -209,7 +209,7 @@ def main(): args.macos = 'm' in args.os # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz'): + if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 756b5fd7c..e0aaafc15 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -32,7 +32,7 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz" +- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz" script: | set -e -o pipefail @@ -90,7 +90,7 @@ script: | BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz # Build dependencies for each host for i in $HOSTS; do diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 21c0d9200..fe677e3a1 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -17,9 +17,9 @@ When complete, it will have produced `Bitcoin-Qt.dmg`. ### Step 1: Obtaining `Xcode.app` Our current macOS SDK -(`Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz`) can be +(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be extracted from -[Xcode_10.2.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). An Apple ID is needed to download this. After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` @@ -31,25 +31,25 @@ approach (tested on Debian Buster) is outlined below: apt install cpio git clone https://github.com/bitcoin-core/apple-sdk-tools.git -# Unpack Xcode_10.2.1.xip and place the resulting Xcode.app in your current +# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current # working directory -python3 apple-sdk-tools/extract_xcode.py -f Xcode_10.2.1.xip | cpio -d -i +python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i ``` On macOS the process is more straightforward: ```bash -xip -x Xcode_10.2.1.xip +xip -x Xcode_11.3.1.xip ``` -### Step 2: Generating `Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` +### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` -To generate `Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz`, run +To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the previous stage) as the first argument. ```bash -# Generate a Xcode-10.2.1-10E1001-extracted-SDK-with-libcxx-headers.tar.gz from +# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from # the supplied Xcode.app ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` @@ -80,7 +80,7 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download [Xcode_10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). This file is many gigabytes in size, but most (but not all) of what we need is contained only in a single directory: diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 272b8230e..1302b4288 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,7 +1,7 @@ OSX_MIN_VERSION=10.12 -OSX_SDK_VERSION=10.14.4 -XCODE_VERSION=10.2.1 -XCODE_BUILD_ID=10E1001 +OSX_SDK_VERSION=10.15.1 +XCODE_VERSION=11.3.1 +XCODE_BUILD_ID=11C505 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ From adf543d7144a44f8cd28a090a21c4e2a606862da Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Mon, 22 Jun 2020 16:46:16 -0400 Subject: [PATCH 8/8] darwin: pass mlinker-version so that clang enables new features Without this clang fails to add any newly-added linker features. Removing this in ca5055a5aa07aba81a87cf12f6f0526a63c423b5 was likely a regression. See https://github.com/bitcoin/bitcoin/pull/19240#issuecomment-647764049 for more discussion. --- depends/hosts/darwin.mk | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 1302b4288..5f0bffa5c 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -2,9 +2,11 @@ OSX_MIN_VERSION=10.12 OSX_SDK_VERSION=10.15.1 XCODE_VERSION=11.3.1 XCODE_BUILD_ID=11C505 +LD64_VERSION=530 + OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ +darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) +darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS)