Merge pull request #7415 from SomberNight/20210718_mac_build

mac build: attempt at "reproducible" codesigned builds

contrib/osx/README.md

@@ -1,9 +1,8 @@
 Building macOS binaries
 =======================
 
-✗ _This script does not produce reproducible output (yet!).
+✓ _This binary should be reproducible, meaning you should be able to generate
-   Please help us remedy this.
+   binaries that match the official releases._
-   [(see #7266)](https://github.com/spesmilo/electrum/issues/7266)_
 
 This guide explains how to build Electrum binaries for macOS systems.
 
@@ -32,6 +31,20 @@ We currently build the release binaries on macOS 10.14.6, and these seem to run
 Before starting, make sure that the Xcode command line tools are installed (e.g. you have `git`).
 
 
+#### Notes about reproducibility
+
+- We recommend creating a VM with a macOS guest, e.g. using VirtualBox,
+  and building there.
+- The guest should run macOS 10.14.6 (that specific version).
+- The unix username should be `vagrant`, and `electrum` should be cloned directly
+  to the user's home dir: `/Users/vagrant/electrum`.
+- Builders need to use the same version of Xcode; and note that
+  full Xcode and Xcode commandline tools differ!
+  You should build with Xcode 11.3.1 (full Xcode).
+- Make sure that you are building from a fresh clone of electrum
+  (or run e.g. `git clean -ffxd` to rm all local changes).
+
+
 #### 1. Get Xcode
 
 Notarizing the application requires full Xcode
@@ -63,3 +76,18 @@ provide these env vars to the `make_osx` script:
     APPLE_ID_USER="me@email.com" \
     APPLE_ID_PASSWORD="1234" \
     ./contrib/osx/make_osx
+
+
+## Verifying reproducibility and comparing against official binary
+
+Every user can verify that the official binary was created from the source code in this 
+repository.
+
+1. Build your own binary as described above.
+2. Use the provided `compare_dmg` script to compare the binary you built with
+   the official release binary.
+    ```
+    $ ./contrib/osx/compare_dmg dist/electrum-*.dmg electrum_dmg_official_release.dmg
+    ```
+   The `compare_dmg` is only needed as the official release binary is codesigned and notarized.
+   Otherwise, the built dmg files should be byte-identical.

contrib/osx/apply_sigs.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+# Copyright (c) 2014-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# This script is based on https://github.com/bitcoin/bitcoin/blob/194b9b8792d9b0798fdb570b79fa51f1d1f5ebaf/contrib/macdeploy/detached-sig-apply.sh
+
+export LC_ALL=C
+set -e
+
+if [ $(uname) != "Darwin" ]; then
+    echo "This script needs to be run on macOS."
+    exit 1
+fi
+
+CP=gcp
+
+UNSIGNED="$1"
+SIGNATURE="$2"
+ARCH=x86_64
+OUTDIR="/tmp/electrum_compare_dmg/signed_app"
+
+if [ -z "$UNSIGNED" ]; then
+    echo "usage: $0 <unsigned app> <path to mac_extracted_sigs.tar.gz>"
+    exit 1
+fi
+
+if [ -z "$SIGNATURE" ]; then
+    echo "usage: $0 <unsigned app> <path to mac_extracted_sigs.tar.gz>"
+    exit 1
+fi
+
+rm -rf ${OUTDIR} && mkdir -p ${OUTDIR}
+${CP} -rf ${UNSIGNED} ${OUTDIR}
+tar xf "${SIGNATURE}" -C ${OUTDIR}
+
+find ${OUTDIR} -name "*.sign" | while read i; do
+    SIZE=$(gstat -c %s "${i}")
+    TARGET_FILE="$(echo "${i}" | sed 's/\.sign$//')"
+
+    if [ -z ${QUIET} ]; then
+        echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}"
+    fi
+    codesign_allocate -i "${TARGET_FILE}" -a ${ARCH} ${SIZE} -o "${i}.tmp"
+
+    OFFSET=$(pagestuff "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g')
+    if [ -z ${QUIET} ]; then
+        echo "Attaching signature at offset ${OFFSET}"
+    fi
+
+    dd if="$i" of="${i}.tmp" bs=1 seek=${OFFSET} count=${SIZE} 2>/dev/null
+    mv "${i}.tmp" "${TARGET_FILE}"
+    rm "${i}"
+    if [ -z ${QUIET} ]; then
+        echo "Success."
+    fi
+done
+echo "Done. .app with sigs applied is at: ${OUTDIR}"

contrib/osx/compare_dmg

@@ -1,28 +1,60 @@
 #!/usr/bin/env bash                                                                                                                                                                                               
 set -e
 
-src_dir=$(dirname "$0")
+if [ $(uname) != "Darwin" ]; then
-cd "$src_dir/../.."
+    echo "This script needs to be run on macOS."
-
+    exit 1
-rm -rf dmg1
+fi
-hdiutil attach $1
+
-cp -r /Volumes/Electrum/Electrum.app/ dmg1
+UNSIGNED_DMG="$1"
-hdiutil detach /Volumes/Electrum
+RELEASE_DMG="$2"
-
+CONTRIB_OSX="$(dirname "$(grealpath "$0")")"
-rm -rf dmg2
+PROJECT_ROOT="$CONTRIB_OSX/../.."
-hdiutil attach $2
+WORKSPACE="/tmp/electrum_compare_dmg"
-cp -r /Volumes/Electrum/Electrum.app/ dmg2
+
-hdiutil detach /Volumes/Electrum
+if [ -z "$UNSIGNED_DMG" ]; then
-
+    echo "usage: $0 <unsigned dmg> <release dmg>"
-# remove signatures
+    exit 1
-for i in $(find dmg1/ ); do codesign --remove-signature $i || true; done;
+fi
-for i in $(find dmg2/ ); do codesign --remove-signature $i || true; done;
+
-
+if [ -z "$RELEASE_DMG" ]; then
-diff=$(diff -qr dmg1 dmg2)
+    echo "usage: $0 <unsigned dmg> <release dmg>"
-echo $diff
+    exit 1
-if [ "$diff" ]
+fi
-then
+
-    echo "failure"
+UNSIGNED_DMG=$(grealpath "$UNSIGNED_DMG")
-else
+RELEASE_DMG=$(grealpath "$RELEASE_DMG")
-    echo "success"
+
+cd "$PROJECT_ROOT"
+rm -rf "$WORKSPACE" && mkdir -p "$WORKSPACE"
+
+DMG_UNSIGNED_UNPACKED="$WORKSPACE/dmg1"
+DMG_RELEASE_UNPACKED="$WORKSPACE/dmg2"
+
+hdiutil attach "$UNSIGNED_DMG"
+cp -r /Volumes/Electrum "$DMG_UNSIGNED_UNPACKED"
+hdiutil detach /Volumes/Electrum
+
+hdiutil attach "$RELEASE_DMG"
+cp -r /Volumes/Electrum "$DMG_RELEASE_UNPACKED"
+hdiutil detach /Volumes/Electrum
+
+# copy signatures from RELEASE_DMG to UNSIGNED_DMG
+echo "Extracting signatures from release app..."
+QUIET="1" "$CONTRIB_OSX/extract_sigs.sh" "$DMG_RELEASE_UNPACKED"/Electrum.app
+echo "Applying extracted signatures to unsigned app..."
+QUIET="1" "$CONTRIB_OSX/apply_sigs.sh" "$DMG_UNSIGNED_UNPACKED"/Electrum.app mac_extracted_sigs.tar.gz
+
+rm mac_extracted_sigs.tar.gz
+
+diff=$(diff -qr "$WORKSPACE/signed_app" "$DMG_RELEASE_UNPACKED") || true
+echo $diff
+if [ "$diff" ]; then
+    echo "DMGs do *not* match."
+    echo "failure"
+    exit 1
+else
+    echo "DMGs match."
+    echo "success"
+    exit 0
 fi

contrib/osx/extract_sigs.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+# Copyright (c) 2014-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# This script is based on https://github.com/bitcoin/bitcoin/blob/194b9b8792d9b0798fdb570b79fa51f1d1f5ebaf/contrib/macdeploy/detached-sig-create.sh
+
+export LC_ALL=C
+set -e
+
+if [ $(uname) != "Darwin" ]; then
+    echo "This script needs to be run on macOS."
+    exit 1
+fi
+
+TEMPDIR="/tmp/electrum_compare_dmg/sigs.temp"
+OUT=mac_extracted_sigs.tar.gz
+OUTROOT=.
+
+if [ -z "$1" ]; then
+    echo "usage: $0 <path to .app>"
+    exit 1
+fi
+BUNDLE="$1"
+BUNDLE_BASENAME=$(basename "$BUNDLE")
+
+rm -rf ${TEMPDIR}
+mkdir -p ${TEMPDIR}
+
+MAYBE_SIGNED_FILES=$(find "$BUNDLE/Contents/MacOS/" -type f)
+
+echo "${MAYBE_SIGNED_FILES}" | while read i; do
+    # skip files where pagestuff errors; these probably do not need signing:
+    pagestuff "$i" -p 1>/dev/null 2>/dev/null || continue
+    TARGETFILE="${BUNDLE_BASENAME}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")"
+    SIZE=$(pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g')
+    OFFSET=$(pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g')
+    SIGNFILE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}.sign"
+    DIRNAME="$(dirname "${SIGNFILE}")"
+    mkdir -p "${DIRNAME}"
+    if [ -z ${QUIET} ]; then
+        echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}"
+    fi
+    dd if="$i" of="${SIGNFILE}" bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null
+done
+
+FILES_TO_COPY=$(cat << EOF
+$BUNDLE/Contents/_CodeSignature/CodeResources
+$BUNDLE/Contents/CodeResources
+EOF
+)
+
+echo "${FILES_TO_COPY}" | while read i; do
+    TARGETFILE="${BUNDLE_BASENAME}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")"
+    RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}"
+    DIRNAME="$(dirname "${RESOURCE}")"
+    mkdir -p "${DIRNAME}"
+    if [ -z ${QUIET} ]; then
+        echo "Adding resource for: \"${TARGETFILE}\""
+    fi
+    cp "${i}" "${RESOURCE}"
+done
+
+tar -C "${TEMPDIR}" -czf "${OUT}" .
+rm -rf "${TEMPDIR}"
+echo "Created ${OUT}"