diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0fc17200212a..ae2e6df8dd1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,19 +37,25 @@ builder-image: image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG variables: SDK_URL: https://bitcoincore.org/depends-sources/sdks - OSX_SDK: "10.14" + XCODE_VERSION: 11.3.1 + XCODE_BUILD_ID: 11C505 MAKEJOBS: -j4 before_script: - echo HOST=$HOST - | - if [ "$HOST" = "x86_64-apple-darwin16" ]; then - echo "Downloading MacOS SDK" + if [ "$HOST" = "x86_64-apple-darwin18" ]; then mkdir -p depends/SDKs mkdir -p depends/sdk-sources - if [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz + OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" + OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + echo "Downloading MacOS SDK" + curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" + fi + if [ -f "$OSX_SDK_PATH" ]; then + echo "Extracting MacOS SDK" + tar -C depends/SDKs -xf "$OSX_SDK_PATH" fi - tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi script: - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS @@ -180,12 +186,12 @@ x86_64-unknown-linux-gnu-release: HOST: x86_64-unknown-linux-gnu DEP_OPTS: "NO_UPNP=1" -x86_64-apple-darwin16: +x86_64-apple-darwin18: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-apple-darwin16 + HOST: x86_64-apple-darwin18 ### @@ -251,7 +257,7 @@ mac-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-apple-darwin16 + - x86_64-apple-darwin18 variables: BUILD_TARGET: mac diff --git a/.travis.yml b/.travis.yml index c4017a236a72..7ea3b7dc67e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -294,9 +294,10 @@ after_success: - stage: test name: 'macOS 10.12 [GOAL: deploy]' env: >- - HOST=x86_64-apple-darwin16 + HOST=x86_64-apple-darwin18 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" - OSX_SDK=10.14 + export XCODE_VERSION=11.3.1 + export XCODE_BUILD_ID=11C505 RUN_UNIT_TESTS=false RUN_FUNCTIONAL_TESTS=false GOAL="deploy" diff --git a/.travis/lint_04_install.sh b/.travis/lint_04_install.sh index 723e7c56f1d9..1b4b6111c4bf 100755 --- a/.travis/lint_04_install.sh +++ b/.travis/lint_04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,3 +9,4 @@ export LC_ALL=C travis_retry pip install codespell==1.13.0 travis_retry pip install flake8==3.5.0 travis_retry pip install vulture==0.29 +travis_retry pip3 install yq diff --git a/.travis/test_05_before_script.sh b/.travis/test_05_before_script.sh index 70caee979c14..f6461c665995 100755 --- a/.travis/test_05_before_script.sh +++ b/.travis/test_05_before_script.sh @@ -7,14 +7,16 @@ export LC_ALL=C.UTF-8 DOCKER_EXEC echo \> \$HOME/.bitcoin # Make sure default datadir does not exist and is never read by creating a dummy file +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" +OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}" mkdir -p depends/SDKs depends/sdk-sources -if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then + curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" fi -if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then + DOCKER_EXEC tar -C "depends/SDKs" -xf "$OSX_SDK_PATH" fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb78ca81fa3d..25b98697090e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ include_directories( if(UNIX AND NOT APPLE) set(DEPENDS_PREFIX depends/x86_64-pc-linux-gnu) elseif(APPLE) - set(DEPENDS_PREFIX depends/x86_64-apple-darwin16) + set(DEPENDS_PREFIX depends/x86_64-apple-darwin18) elseif(WIN32) set(DEPENDS_PREFIX depends/x86_64-w64-mingw32) endif() diff --git a/ci/Dockerfile.builder b/ci/Dockerfile.builder index 89cf25c8ca21..7f7a611acbfd 100644 --- a/ci/Dockerfile.builder +++ b/ci/Dockerfile.builder @@ -18,6 +18,7 @@ RUN pip3 install jinja2 RUN pip3 install flake8==3.5.0 RUN pip3 install codespell==1.13.0 RUN pip3 install vulture==0.29 +RUN pip3 install yq # dash_hash RUN git clone https://github.com/dashpay/dash_hash diff --git a/ci/build_depends.sh b/ci/build_depends.sh index ecacf6807d72..1b0f42e5a6b2 100755 --- a/ci/build_depends.sh +++ b/ci/build_depends.sh @@ -19,12 +19,14 @@ ln -s $CACHE_DIR/sdk-sources depends/sdk-sources mkdir -p depends/SDKs -if [ -n "$OSX_SDK" ]; then - if [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +if [ -n "$XCODE_VERSION" ]; then + OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" + OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" fi - if [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz + if [ -f "$OSX_SDK_PATH" ]; then + tar -C depends/SDKs -xf "$OSX_SDK_PATH" fi fi diff --git a/ci/matrix.sh b/ci/matrix.sh index 22c200545c4a..adaadbac26e2 100755 --- a/ci/matrix.sh +++ b/ci/matrix.sh @@ -76,9 +76,10 @@ elif [ "$BUILD_TARGET" = "linux64_release" ]; then export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports" export PYZMQ=true elif [ "$BUILD_TARGET" = "mac" ]; then - export HOST=x86_64-apple-darwin16 + export HOST=x86_64-apple-darwin18 export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --disable-miner --enable-werror" - export OSX_SDK=10.14 + export XCODE_VERSION=11.3.1 + export XCODE_BUILD_ID=11C505 export GOAL="all deploy" export RUN_UNITTESTS=false export RUN_INTEGRATIONTESTS=false diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 79db87436638..ef1733daa7b6 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -71,8 +71,8 @@ def build(): if args.macos: print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/MacOSX10.14.sdk.tar.gz']) - subprocess.check_output(["echo '328aff47e28c17093d59a72712a9b2e62cd8a8b87bbe03f91abb32960b413f0f inputs/MacOSX10.14.sdk.tar.gz' | sha256sum -c"], shell=True) + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz']) + subprocess.check_output(["echo '436df6dfc7073365d12f8ef6c1fdb060777c720602cc67c2dcf9a59d94290e38 inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml']) subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index b7be3f4c8f5a..da7cd35b92d3 100755 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -50,7 +50,7 @@ script: | export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -128,7 +128,7 @@ script: | cat << EOF > ${WRAP_DIR}/${prog} #!/usr/bin/env bash # GCCVERSION=${GCCVERSION} - REAL="`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1`" + REAL="$(which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1)" for var in "\$@" do if [ "\$var" = "-m32" ]; then @@ -144,7 +144,7 @@ script: | done cd dash - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" @@ -163,10 +163,11 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ + CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ make dist - SOURCEDIST=`echo dashcore-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + SOURCEDIST=$(echo dashcore-*.tar.gz) + DISTNAME=${SOURCEDIST/%.tar.gz} + # Correct tar file order mkdir -p temp pushd temp @@ -183,7 +184,7 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index e46699551026..9f9e2795dac4 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -16,7 +16,7 @@ script: | WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} - export PATH=`pwd`:$PATH + export PATH="$PWD":$PATH FAKETIME_PROGS="dmg genisoimage" # Create global faketime wrappers diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index f1a72a8c9e15..4b1138ae4d51 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -31,12 +31,12 @@ remotes: - "url": "https://github.com/dashpay/dash.git" "dir": "dash" files: -- "MacOSX10.14.sdk.tar.gz" +- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz" script: | set -e -o pipefail WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin16" + HOSTS="x86_64-apple-darwin18" CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage" @@ -46,7 +46,7 @@ script: | export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -104,10 +104,10 @@ script: | export PATH=${WRAP_DIR}:${PATH} cd dash - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz # Build dependencies for each host for i in $HOSTS; do @@ -122,10 +122,10 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ + CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ make dist - SOURCEDIST=`echo dashcore-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + SOURCEDIST=$(echo dashcore-*.tar.gz) + DISTNAME=${SOURCEDIST/%.tar.gz} # Correct tar file order mkdir -p temp @@ -143,7 +143,7 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST @@ -186,8 +186,8 @@ script: | done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src - mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin16.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz - mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin16-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz + mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin18.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz + mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin18-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz # Compress ccache (otherwise the assert file will get too huge) if [ "$CCACHE_DIR" != "" ]; then diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index fd08214aa715..7d2092855c80 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -19,7 +19,7 @@ files: script: | set -e -o pipefail - BUILD_DIR=`pwd` + BUILD_DIR="$PWD" SIGDIR=${BUILD_DIR}/signature/win UNSIGNED_DIR=${BUILD_DIR}/unsigned @@ -35,7 +35,7 @@ script: | ./configure --without-gsf --without-curl --disable-dependency-tracking make find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do - INFILE="`basename "${i}"`" - OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`" + INFILE="$(basename "${i}")" + OUTFILE="${INFILE/%-unsigned}" ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem" done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index f344dbe88ad4..2202d5ff628d 100755 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -43,7 +43,7 @@ script: | export GZIP="-9n" export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -137,7 +137,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} cd dash - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" @@ -152,10 +152,10 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ + CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ make dist - SOURCEDIST=`echo dashcore-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + SOURCEDIST=$(echo dashcore-*.tar.gz) + DISTNAME=${SOURCEDIST/%.tar.gz} # Correct tar file order mkdir -p temp @@ -175,7 +175,7 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index ae7b57b96f1e..53ab880d01c6 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -14,51 +14,44 @@ When complete, it will have produced `Dash-Qt.dmg`. ## SDK Extraction -`Xcode.app` is packaged in a `.xip` archive. -This makes the SDK less-trivial to extract on non-macOS machines. -One approach (tested on Debian Buster) is outlined below: +### Step 1: Obtaining `Xcode.app` -```bash - -apt install clang cpio git liblzma-dev libxml2-dev libssl-dev make - -git clone https://github.com/tpoechtrager/xar -pushd xar/xar -./configure -make -make install -popd - -git clone https://github.com/NiklasRosenstein/pbzx -pushd pbzx -clang -llzma -lxar pbzx.c -o pbzx -Wl,-rpath=/usr/local/lib -popd +Our current macOS SDK +(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be +extracted from +[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). +An Apple ID is needed to download this. -xar -xf Xcode_10.2.1.xip -C . +After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` +archive. This makes the SDK less-trivial to extract on non-macOS machines. One +approach (tested on Debian Buster) is outlined below: -./pbzx/pbzx -n Content | cpio -i +```bash +# Install/clone tools needed for extracting Xcode.app +apt install cpio +git clone https://github.com/bitcoin-core/apple-sdk-tools.git -find Xcode.app -type d -name MacOSX.sdk -execdir sh -c 'tar -c MacOSX.sdk/ | gzip -9n > /MacOSX10.14.sdk.tar.gz' \; +# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current +# working directory +python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i ``` -on macOS the process is more straightforward: +On macOS the process is more straightforward: ```bash -xip -x Xcode_10.2.1.xip -tar -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk +xip -x Xcode_11.3.1.xip ``` -Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from -[Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). -The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First -ensure the DMG file is in the current directory, and then run the script. You -may wish to delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the -directory) when you've confirmed the extraction succeeded. +### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` + +To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run +the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the +previous stage) as the first argument. ```bash -apt-get install p7zip-full sleuthkit -contrib/macdeploy/extract-osx-sdk.sh -rm -rf 5.hfs MacOSX10.11.sdk +# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from +# the supplied Xcode.app +./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` ## Deterministic macOS DMG Notes @@ -87,13 +80,13 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download [Xcode 10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). This file is many gigabytes in size, but most (but not all) of what we need is contained only in a single directory: ```bash -Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk +Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk ``` See the SDK Extraction notes above for how to obtain it. @@ -130,7 +123,6 @@ deterministic. Here's how it works: of a tarball, which also contains all of the tools that have been previously (deterministically) built in order to create a final DMG. - The Apple keyholder uses this unsigned app to create a detached signature, using the - script that is also included there. Detached signatures are available. + script that is also included there. Detached signatures are available - Builders feed the unsigned app + detached signature back into Gitian. It uses the pre-built tools to recombine the pieces into a deterministic DMG. - diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh index f68815f9b311..b7ac0852bba9 100755 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ b/contrib/macdeploy/extract-osx-sdk.sh @@ -8,7 +8,7 @@ set -e INPUTFILE="Xcode_7.3.1.dmg" HFSFILENAME="5.hfs" -SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk" +SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz" 7z x "${INPUTFILE}" "${HFSFILENAME}" SDKNAME="$(basename "${SDKDIR}")" diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk new file mode 100755 index 000000000000..457d8f5e645c --- /dev/null +++ b/contrib/macdeploy/gen-sdk @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import argparse +import plistlib +import pathlib +import sys +import tarfile +import gzip +import os +import contextlib + +@contextlib.contextmanager +def cd(path): + """Context manager that restores PWD even if an exception was raised.""" + old_pwd = os.getcwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(old_pwd) + +def run(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1) + parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False) + + args = parser.parse_args() + + xcode_app = pathlib.Path(args.xcode_app[0]).resolve() + assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app) + + xcode_app_plist = xcode_app.joinpath("Contents/version.plist") + with xcode_app_plist.open('rb') as fp: + pl = plistlib.load(fp) + xcode_version = pl['CFBundleShortVersionString'] + xcode_build_id = pl['ProductBuildVersion'] + print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)) + + sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") + sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist") + with sdk_plist.open('rb') as fp: + pl = plistlib.load(fp) + sdk_version = pl['ProductVersion'] + sdk_build_id = pl['ProductBuildVersion'] + print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id)) + + out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) + + xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1") + assert xcode_libcxx_dir.is_dir() + + if args.out_sdktgz: + out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) + else: + # Construct our own out_sdktgz if not specified on the command line + out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) + + def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): + """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files' + names + + e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking: + + tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir") + + would result in the following members being added to tarfp: + + foo/bar/ -> corresponding to /root/bazdir + foo/bar/qux -> corresponding to /root/bazdir/qux + + """ + def change_tarinfo_base(tarinfo): + if tarinfo.name and tarinfo.name.startswith("./"): + tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) + if tarinfo.linkname and tarinfo.linkname.startswith("./"): + tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + return tarinfo + with cd(dir_to_add): + tarfp.add(".", recursive=True, filter=change_tarinfo_base) + + print("Creating output .tar.gz file...") + with out_sdktgz_path.open("wb") as fp: + with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: + with tarfile.open(mode="w", fileobj=gzf) as tarfp: + print("Adding MacOSX SDK {} files...".format(sdk_version)) + tarfp_add_with_base_change(tarfp, sdk_dir, out_name) + print("Adding libc++ headers...") + tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name)) + print("Done! Find the resulting gzipped tarball at:") + print(out_sdktgz_path.resolve()) + +if __name__ == '__main__': + run() diff --git a/depends/Makefile b/depends/Makefile index c5cba92edf55..6d3b1d852d88 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -116,6 +116,13 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) +ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +# Make sure that cache is invalidated when switching between system and +# depends-managed, pinned clang +build_id_string+=system_clang +$(host_arch)_$(host_os)_id_string+=system_clang +endif + qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) wallet_packages_$(NO_WALLET) = $(wallet_packages) upnp_packages_$(NO_UPNP) = $(upnp_packages) @@ -131,11 +138,11 @@ all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk +$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils) $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk -toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) @@ -146,15 +153,39 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ +# $PATH is not preserved between ./configure and make by convention. Its +# modification and overriding at ./configure time is (as I understand it) +# supposed to be captured by the AC_{PROG_{,OBJ}CXX,PATH_{PROG,TOOL}} macros, +# which will expand the program names to their full absolute paths. The notable +# exception is command line overriding: ./configure CC=clang, which skips the +# program name expansion step, and works because the user implicitly indicates +# with CC=clang that clang will be available in $PATH at all times, and is most +# likely part of the user's system. +# +# Therefore, when we "seed the autoconf cache"/"override well-known program +# vars" by setting AR= in our config.site, either one of two things needs +# to be true for the build system to work correctly: +# +# 1. If we refer to the program by name (e.g. AR=riscv64-gnu-linux-ar), the +# tool needs to be available in $PATH at all times. +# +# 2. If the tool is _**not**_ expected to be available in $PATH at all times +# (such as is the case for our native_cctools binutils tools), it needs to +# be referred to by its absolute path, such as would be output by the +# AC_PATH_{PROG,TOOL} macros. +# +# Minor note: it is also okay to refer to tools by their absolute path even if +# we expect them to be available in $PATH at all times, more specificity does +# not hurt. $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) $(AT)@mkdir -p $(@D) $(AT)sed -e 's|@HOST@|$(host)|' \ - -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ - -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ - -e 's|@AR@|$(toolchain_path)$(host_AR)|' \ - -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \ - -e 's|@NM@|$(toolchain_path)$(host_NM)|' \ - -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \ + -e 's|@CC@|$(host_CC)|' \ + -e 's|@CXX@|$(host_CXX)|' \ + -e 's|@AR@|$(host_AR)|' \ + -e 's|@RANLIB@|$(host_RANLIB)|' \ + -e 's|@NM@|$(host_NM)|' \ + -e 's|@STRIP@|$(host_STRIP)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ diff --git a/depends/README.md b/depends/README.md index b2c61b5787e4..c971a042fa6b 100644 --- a/depends/README.md +++ b/depends/README.md @@ -21,7 +21,7 @@ created. To use it for Dash: Common `host-platform-triplets` for cross compilation are: - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin16` for macOS +- `x86_64-apple-darwin18` for macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit - `riscv32-linux-gnu` for Linux RISC-V 32 bit @@ -52,19 +52,34 @@ RISC-V known issue: gcc-7.3.0 and gcc-7.3.1 result in a broken `test_dash` execu this is apparently fixed in gcc-8.1.0. Dependency Options: -The following can be set when running make: make FOO=bar - - SOURCES_PATH: downloaded sources will be placed here - BASE_CACHE: built packages will be placed here - SDK_PATH: Path where sdk's can be found (used by macOS) - FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up - NO_QT: Don't download/build/cache qt and its dependencies - NO_WALLET: Don't download/build/cache libs needed to enable the wallet - NO_UPNP: Don't download/build/cache packages needed for enabling upnp - DEBUG: disable some optimizations and enable more runtime checking - HOST_ID_SALT: Optional salt to use when generating host package ids - BUILD_ID_SALT: Optional salt to use when generating build package ids - +The following can be set when running make: `make FOO=bar` + +
+
SOURCES_PATH
+
downloaded sources will be placed here
+
BASE_CACHE
+
built packages will be placed here
+
SDK_PATH
+
Path where sdk's can be found (used by macOS)
+
FALLBACK_DOWNLOAD_PATH
+
If a source file can't be fetched, try here before giving up
+
NO_QT
+
Don't download/build/cache qt and its dependencies
+
NO_WALLET
+
Don't download/build/cache libs needed to enable the wallet
+
NO_UPNP
+
Don't download/build/cache packages needed for enabling upnp
+
DEBUG
+
disable some optimizations and enable more runtime checking
+
HOST_ID_SALT
+
Optional salt to use when generating host package ids
+
BUILD_ID_SALT
+
Optional salt to use when generating build package ids
+
FORCE_USE_SYSTEM_CLANG
+
(EXPERTS ONLY) When cross-compiling for macOS, use clang found in the +system's $PATH rather than the default prebuilt release of clang +from llvm.org
+
If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to Dash Core's configure. In this case, `--disable-wallet`. diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 4d538dcf4a39..92addcfb8895 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -19,4 +19,5 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool) darwin_OTOOL:=$(shell xcrun -f otool) darwin_NM:=$(shell xcrun -f nm) darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +darwin_native_binutils= darwin_native_toolchain= diff --git a/depends/config.site.in b/depends/config.site.in index ef865bbe17fd..8bea2469441a 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -1,4 +1,14 @@ -depends_prefix="`dirname ${ac_site_file}`/.." +# shellcheck shell=sh disable=SC2034 # Many variables set will be used in + # ./configure but shellcheck doesn't know + # that, hence: disable=SC2034 + +true # Dummy command because shellcheck treats all directives before first + # command as file-wide, and we only want to disable for one line. + # + # See: https://github.com/koalaman/shellcheck/wiki/Directive + +# shellcheck disable=SC2154 +depends_prefix="$(cd "$(dirname ${ac_site_file})/.." && pwd)" cross_compiling=maybe host_alias=@HOST@ @@ -43,7 +53,7 @@ if test x@host_os@ = xdarwin; then fi PATH=$depends_prefix/native/bin:$PATH -PKG_CONFIG="`which pkg-config` --static" +PKG_CONFIG="$(which pkg-config) --static" # These two need to remain exported because pkg-config does not see them # otherwise. That means they must be unexported at the end of configure.ac to @@ -61,7 +71,7 @@ fi if test -n "@CXX@" -a -z "${CXX}"; then CXX="@CXX@" fi -PYTHONPATH=$depends_prefix/native/lib/python3/dist-packages:$PYTHONPATH +PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PATH_SEPARATOR}}${PYTHONPATH}" if test -n "@AR@"; then AR=@AR@ diff --git a/depends/funcs.mk b/depends/funcs.mk index 472234287b2a..32032b9cb7f0 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -1,17 +1,23 @@ define int_vars #Set defaults for vars which may be overridden per-package -$(1)_cc=$($($(1)_type)_CC) -$(1)_cxx=$($($(1)_type)_CXX) -$(1)_objc=$($($(1)_type)_OBJC) -$(1)_objcxx=$($($(1)_type)_OBJCXX) -$(1)_ar=$($($(1)_type)_AR) -$(1)_ranlib=$($($(1)_type)_RANLIB) -$(1)_libtool=$($($(1)_type)_LIBTOOL) -$(1)_nm=$($($(1)_type)_NM) -$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS) -$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS) -$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib -$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include +$(1)_cc=$$($$($(1)_type)_CC) +$(1)_cxx=$$($$($(1)_type)_CXX) +$(1)_objc=$$($$($(1)_type)_OBJC) +$(1)_objcxx=$$($$($(1)_type)_OBJCXX) +$(1)_ar=$$($$($(1)_type)_AR) +$(1)_ranlib=$$($$($(1)_type)_RANLIB) +$(1)_libtool=$$($$($(1)_type)_LIBTOOL) +$(1)_nm=$$($$($(1)_type)_NM) +$(1)_cflags=$$($$($(1)_type)_CFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CFLAGS) +$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CXXFLAGS) +$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \ + $$($$($(1)_type)_$$(release_type)_LDFLAGS) \ + -L$$($($(1)_type)_prefix)/lib +$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \ + -I$$($$($(1)_type)_prefix)/include $(1)_recipe_hash:= endef @@ -43,7 +49,7 @@ endef define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) -$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies))) +$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) @@ -253,4 +259,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$ $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package)))) #special exception: if a toolchain package exists, all non-native packages depend on it -$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) )) +$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) )) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 1bc4fb81896f..ee4a45154bdb 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,8 +1,119 @@ -OSX_MIN_VERSION=10.12 -OSX_SDK_VERSION=10.14 -OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ +OSX_MIN_VERSION=10.14 +OSX_SDK_VERSION=10.15.1 +XCODE_VERSION=11.3.1 +XCODE_BUILD_ID=11C505 +LD64_VERSION=530 + +OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers + +darwin_native_binutils=native_cctools + +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang +# from llvm.org + +# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG +# is empty +darwin_native_toolchain=native_cctools + +clang_prog=$(build_prefix)/bin/clang +clangxx_prog=$(clang_prog)++ + +clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version) +else +# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's +# system + +darwin_native_toolchain= + +# We can't just use $(shell command -v clang) because GNU Make handles builtins +# in a special way and doesn't know that `command` is a POSIX-standard builtin +# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. +# At the time of writing, GNU Make v4.2.1 is still being used in supported +# distro releases. +# +# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html +clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") +clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") + +clang_resource_dir=$(shell clang -print-resource-dir) +endif + +cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL + +# Make-only lowercase function +lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) + +# For well-known tools provided by cctools, make sure that their well-known +# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG} +# would. +$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL)))) + +# Flag explanations: +# +# -mlinker-version +# +# Ensures that modern linker features are enabled. See here for more +# details: https://github.com/bitcoin/bitcoin/pull/19407. +# +# -B$(build_prefix)/bin +# +# Explicitly point to our binaries (e.g. cctools) so that they are +# ensured to be found and preferred over other possibilities. +# +# -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 +# +# Forces clang to use the libc++ headers from our SDK and completely +# forget about the libc++ headers from the standard directories +# +# TODO: Once we start requiring a clang version that has the +# -stdlib++-isystem flag first introduced here: +# https://reviews.llvm.org/D64089, we should use that instead. Read the +# differential summary there for more details. +# +# -Xclang -*system \ +# -Xclang -*system \ +# -Xclang -*system ... +# +# Adds path_a, path_b, and path_c to the bottom of clang's list of +# include search paths. This is used to explicitly specify the list of +# system include search paths and its ordering, rather than rely on +# clang's autodetection routine. This routine has been shown to: +# 1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1 +# when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034) +# 2. Fail to pickup C headers in $SYSROOT/usr/include when +# C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9) +# +# Talking directly to cc1 with -Xclang here grants us access to specify +# more granular categories for these system include search paths, and we +# can use the correct categories that these search paths would have been +# placed in if the autodetection routine had worked correctly. (see: +# https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment) +# +# Furthermore, it places these search paths after any "non-Xclang" +# specified search paths. This prevents any additional clang options or +# environment variables from coming after or in between these system +# include search paths, as that would be wrong in general but would also +# break #include_next's. +# +darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + --sysroot=$(OSX_SDK) \ + -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include +darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + --sysroot=$(OSX_SDK) \ + -stdlib=libc++ -nostdinc++ \ + -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \ + -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) @@ -12,5 +123,3 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) - -darwin_native_toolchain=native_cctools diff --git a/depends/packages/bls-dash.mk b/depends/packages/bls-dash.mk index 2ff05466c346..de40797287a5 100644 --- a/depends/packages/bls-dash.mk +++ b/depends/packages/bls-dash.mk @@ -7,6 +7,7 @@ $(package)_build_subdir=build $(package)_sha256_hash=276c8573104e5f18bb5b9fd3ffd49585dda5ba5f6de2de74759dda8ca5a9deac $(package)_dependencies=gmp cmake $(package)_patches=gcc_alignment_cast.patch +$(package)_darwin_triplet=x86_64-apple-darwin18 $(package)_relic_version=3a23142be0a5510a3aa93cd6c76fc59d3fc732a5 $(package)_relic_download_path=https://github.com/relic-toolkit/relic/archive @@ -46,8 +47,9 @@ define $(package)_set_vars $(package)_config_opts_debug=-DDEBUG=ON -DCMAKE_BUILD_TYPE=Debug ifneq ($(darwin_native_toolchain),) - $(package)_config_opts_darwin+= -DCMAKE_AR="$(host_prefix)/native/bin/$($(package)_ar)" - $(package)_config_opts_darwin+= -DCMAKE_RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)" + $(package)_config_opts_darwin+= -DCMAKE_AR="$(host_prefix)/native/bin/$($(package)_darwin_triplet)-ar" + $(package)_config_opts_darwin+= -DCMAKE_LINKER="$(host_prefix)/native/bin/$($(package)_darwin_triplet)-ld" + $(package)_config_opts_darwin+= -DCMAKE_RANLIB="$(host_prefix)/native/bin/$($(package)_darwin_triplet)-ranlib" endif $(package)_cppflags+=-UBLSALLOC_SODIUM diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 4195230b4003..072d3828a670 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,14 +1,16 @@ package=native_cctools -$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f +$(package)_version=4da2f3b485bcf4cef526f30c0b8c0bcda99cdbb4 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04 +$(package)_sha256_hash=a2d491c0981cef72fee2b833598f20f42a6c44a7614a61c439bda93d56446fec $(package)_build_subdir=cctools -$(package)_clang_version=6.0.1 +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +$(package)_clang_version=8.0.0 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0 +$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0 +endif $(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive @@ -16,15 +18,25 @@ $(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz $(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz $(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3 -$(package)_extra_sources=$($(package)_clang_file_name) -$(package)_extra_sources += $($(package)_libtapi_file_name) +$(package)_extra_sources=$($(package)_libtapi_file_name) +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +$(package)_extra_sources += $($(package)_clang_file_name) +endif +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) endef +else +define $(package)_fetch_cmds +$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) +endef +endif +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_extract_cmds mkdir -p $($(package)_extract_dir) && \ echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ @@ -38,12 +50,28 @@ define $(package)_extract_cmds rm -f toolchain/lib/libc++abi.so* && \ tar --no-same-owner --strip-components=1 -xf $($(package)_source) endef +else +define $(package)_extract_cmds + mkdir -p $($(package)_extract_dir) && \ + echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_libtapi_sha256_hash) $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + mkdir -p libtapi && \ + tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \ + tar --no-same-owner --strip-components=1 -xf $($(package)_source) +endef +endif define $(package)_set_vars $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir) $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib + ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ + else + $(package)_cc=clang + $(package)_cxx=clang++ + endif endef define $(package)_preprocess_cmds @@ -60,6 +88,7 @@ define $(package)_build_cmds $(MAKE) endef +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ @@ -72,7 +101,13 @@ define $(package)_stage_cmds cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ - cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ - if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ - if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi + cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil +endef +else +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ + mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ + cd $($(package)_extract_dir) && \ + cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ endef +endif diff --git a/doc/build-cross.md b/doc/build-cross.md index cb737fc0a608..a11a7b3e69db 100644 --- a/doc/build-cross.md +++ b/doc/build-cross.md @@ -19,20 +19,20 @@ Additionally, the Mac OSX SDK must be downloaded and extracted manually: ```bash $ mkdir -p depends/sdk-sources $ mkdir -p depends/SDKs -$ curl https://bitcoincore.org/depends-sources/sdks/MacOSX10.14.sdk.tar.gz -o depends/sdk-sources/MacOSX10.14.sdk.tar.gz -$ tar -C depends/SDKs -xf depends/sdk-sources/MacOSX10.14.sdk.tar.gz +$ curl https://bitcoincore.org/depends-sources/sdks/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz -o depends/sdk-sources/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz +$ tar -C depends/SDKs -xf depends/sdk-sources/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz ``` When building the dependencies, as described in [build-generic](build-generic.md), use ```bash -$ make HOST=x86_64-apple-darwin16 -j4 +$ make HOST=x86_64-apple-darwin18 -j4 ``` When building Dash Core, use ```bash -$ ./configure --prefix=`pwd`/depends/x86_64-apple-darwin16 +$ ./configure --prefix=`pwd`/depends/x86_64-apple-darwin18 ``` Windows 64bit Cross-compilation diff --git a/doc/build-generic.md b/doc/build-generic.md index 2e6a543d45ff..e0fdf1b1afd9 100644 --- a/doc/build-generic.md +++ b/doc/build-generic.md @@ -48,7 +48,7 @@ Please replace `` with your local system's `host-platform-triplet`. The fo - `i686-pc-linux-gnu` for Linux32 - `x86_64-pc-linux-gnu` for Linux64 - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin16` for MacOSX +- `x86_64-apple-darwin18` for MacOSX - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index 12c661d016d1..bb6c0e9bf3ad 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ LSMinimumSystemVersion - 10.12.0 + 10.14.0 LSArchitecturePriority diff --git a/src/random.cpp b/src/random.cpp index b356c59d426c..e1144524b34b 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -13,7 +13,7 @@ #endif #include // for LogPrint() #include // for WAIT_LOCK -#include // for GetTime() +#include // for GetTimeMicros() #include #include @@ -248,13 +248,10 @@ void GetOSRand(unsigned char *ent32) // Silence a compiler warning about unused function. (void)GetDevURandom; #elif defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) - // We need a fallback for OSX < 10.12 - if (&getentropy != nullptr) { - if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { - RandFailure(); - } - } else { - GetDevURandom(ent32); + /* getentropy() is available on macOS 10.12 and later. + */ + if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { + RandFailure(); } // Silence a compiler warning about unused function. (void)GetDevURandom; diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index cf8a37c3a108..f15324d4c17e 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -16,11 +16,6 @@ if [ "$TRAVIS" = "true" ]; then unset LC_ALL fi -if ! command -v shellcheck > /dev/null; then - echo "Skipping shell linting since shellcheck is not installed." - exit 0 -fi - # Disabled warnings: disabled=( SC1087 # Use braces when expanding arrays, e.g. ${array[idx]} (or ${var}[.. to quiet). @@ -46,5 +41,40 @@ disabled=( SC2230 # which is non-standard. Use builtin 'command -v' instead. SC2236 # Don't force -n instead of ! -z. ) -shellcheck -e "$(IFS=","; echo "${disabled[*]}")" \ - $(git ls-files -- "*.sh" | grep -vE 'src/(secp256k1|univalue)/') +disabled_gitian=( + SC2094 # Make sure not to read and write the same file in the same pipeline. + SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects. + SC2230 # which is non-standard. Use builtin 'command -v' instead. +) + +EXIT_CODE=0 + +if ! command -v shellcheck > /dev/null; then + echo "Skipping shell linting since shellcheck is not installed." + exit $EXIT_CODE +fi + +EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" +SOURCED_FILES=$(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') # Check shellcheck directive used for sourced files +if ! shellcheck "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then + EXIT_CODE=1 +fi + +if ! command -v yq > /dev/null; then + echo "Skipping Gitian desriptor scripts checking since yq is not installed." + exit $EXIT_CODE +fi + +EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" +for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') +do + echo + echo "$descriptor" + # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. + SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor") + if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then + EXIT_CODE=1 + fi +done + +exit $EXIT_CODE