android build: attempt at reproducible builds

2021-04-30 16:11:09 +02:00
parent c3ccfd7d19
commit 3c9346cce2
16 changed files with 336 additions and 60 deletions
--- a/contrib/android/Dockerfile
+++ b/contrib/android/Dockerfile
@@ -1,6 +1,6 @@
 # based on https://github.com/kivy/python-for-android/blob/master/Dockerfile

-FROM ubuntu:20.04
+FROM ubuntu:20.04@sha256:86ac87f73641c920fb42cc9612d4fb57b5626b56ea2a19b894d0673fd5b4f2e9

 ENV DEBIAN_FRONTEND=noninteractive

@@ -20,7 +20,8 @@ RUN apt -y update -qq \


 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk"
-ENV ANDROID_NDK_VERSION="19c"
+ENV ANDROID_NDK_VERSION="22b"
+ENV ANDROID_NDK_HASH="ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}"

 # get the latest version from https://developer.android.com/ndk/downloads/index.html
@@ -31,6 +32,7 @@ ENV ANDROID_NDK_DL_URL="https://dl.google.com/android/repository/${ANDROID_NDK_A
 RUN curl --location --progress-bar \
        "${ANDROID_NDK_DL_URL}" \
        --output "${ANDROID_NDK_ARCHIVE}" \
+    && echo "${ANDROID_NDK_HASH} ${ANDROID_NDK_ARCHIVE}" | sha256sum -c - \
    && mkdir --parents "${ANDROID_NDK_HOME_V}" \
    && unzip -q "${ANDROID_NDK_ARCHIVE}" -d "${ANDROID_HOME}" \
    && ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" \
@@ -42,6 +44,7 @@ ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk"
 # get the latest version from https://developer.android.com/studio/index.html
 ENV ANDROID_SDK_TOOLS_VERSION="6514223"
 ENV ANDROID_SDK_BUILD_TOOLS_VERSION="29.0.3"
+ENV ANDROID_SDK_HASH="ef319a5afdb41822cb1c88d93bc7c23b0af4fc670abca89ff0346ee6688da797"
 ENV ANDROID_SDK_TOOLS_ARCHIVE="commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip"
 ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}"
 ENV ANDROID_SDK_MANAGER="${ANDROID_SDK_HOME}/tools/bin/sdkmanager --sdk_root=${ANDROID_SDK_HOME}"
@@ -50,6 +53,7 @@ ENV ANDROID_SDK_MANAGER="${ANDROID_SDK_HOME}/tools/bin/sdkmanager --sdk_root=${A
 RUN curl --location --progress-bar \
        "${ANDROID_SDK_TOOLS_DL_URL}" \
        --output "${ANDROID_SDK_TOOLS_ARCHIVE}" \
+    && echo "${ANDROID_SDK_HASH} ${ANDROID_SDK_TOOLS_ARCHIVE}" | sha256sum -c - \
    && mkdir --parents "${ANDROID_SDK_HOME}" \
    && unzip -q "${ANDROID_SDK_TOOLS_ARCHIVE}" -d "${ANDROID_SDK_HOME}" \
    && rm -rf "${ANDROID_SDK_TOOLS_ARCHIVE}"
@@ -61,7 +65,8 @@ RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \

 # accept Android licenses (JDK necessary!)
 RUN apt -y update -qq \
-    && apt -y install -qq --no-install-recommends openjdk-13-jdk \
+    && apt -y install -qq --no-install-recommends \
+        openjdk-11-jdk-headless \
    && apt -y autoremove
 RUN yes | ${ANDROID_SDK_MANAGER} --licenses > /dev/null

@@ -74,14 +79,16 @@ RUN ${ANDROID_SDK_MANAGER} "platforms;android-24" > /dev/null && \

 # download ANT
 ENV APACHE_ANT_VERSION="1.9.4"
+ENV APACHE_ANT_HASH="66d3edcbb0eba11387705cd89178ffb65e55cd53f13ca35c1bb983c0f9992540"
 ENV APACHE_ANT_ARCHIVE="apache-ant-${APACHE_ANT_VERSION}-bin.tar.gz"
-ENV APACHE_ANT_DL_URL="http://archive.apache.org/dist/ant/binaries/${APACHE_ANT_ARCHIVE}"
+ENV APACHE_ANT_DL_URL="https://archive.apache.org/dist/ant/binaries/${APACHE_ANT_ARCHIVE}"
 ENV APACHE_ANT_HOME="${ANDROID_HOME}/apache-ant"
 ENV APACHE_ANT_HOME_V="${APACHE_ANT_HOME}-${APACHE_ANT_VERSION}"

 RUN curl --location --progress-bar \
        "${APACHE_ANT_DL_URL}" \
        --output "${APACHE_ANT_ARCHIVE}" \
+    && echo "${APACHE_ANT_HASH} ${APACHE_ANT_ARCHIVE}" | sha256sum -c - \
    && tar -xf "${APACHE_ANT_ARCHIVE}" -C "${ANDROID_HOME}" \
    && ln -sfn "${APACHE_ANT_HOME_V}" "${APACHE_ANT_HOME}" \
    && rm -rf "${APACHE_ANT_ARCHIVE}"
@@ -94,6 +101,7 @@ ENV WORK_DIR="${HOME_DIR}/wspace" \

 # install system/build dependencies
 # https://github.com/kivy/buildozer/blob/master/docs/source/installation.rst#android-on-ubuntu-2004-64bit
+# TODO probably need to pin versions of at least some of these for over-time reproducibility?
 RUN apt -y update -qq \
    && apt -y install -qq --no-install-recommends \
        python3 \
@@ -110,7 +118,6 @@ RUN apt -y update -qq \
        unzip \
        build-essential \
        ccache \
-        openjdk-13-jdk \
        autoconf \
        libtool \
        pkg-config \
@@ -144,15 +151,9 @@ RUN chown ${USER} /opt
 USER ${USER}


-RUN python3 -m pip install --user --upgrade pip
-RUN python3 -m pip install --user --upgrade wheel
-RUN python3 -m pip install --user --upgrade cython==0.29.19
-RUN python3 -m pip install --user --pre kivy
-RUN python3 -m pip install --user image
-
-# prepare git
-RUN git config --global user.name "John Doe" \
-    && git config --global user.email johndoe@example.com
+COPY requirements-build-android.txt /opt/deterministic-build/
+RUN python3 -m pip install --no-dependencies --user \
+    -r /opt/deterministic-build/requirements-build-android.txt

 # install buildozer
 RUN cd /opt \
@@ -161,8 +162,8 @@ RUN cd /opt \
    && git remote add sombernight https://github.com/SomberNight/buildozer \
    && git fetch --all \
    # commit: from branch sombernight/electrum_20210421
-    && git checkout "c17ac3618334c9936253e8f5b88dce43dc4da75b^{commit}" \
-    && python3 -m pip install --user -e .
+    && git checkout "d570116e88184b0eca0c6b59a25edd49d977da23^{commit}" \
+    && python3 -m pip install --no-dependencies --user -e .

 # install python-for-android
 RUN cd /opt \
@@ -170,9 +171,9 @@ RUN cd /opt \
    && cd python-for-android \
    && git remote add sombernight https://github.com/SomberNight/python-for-android \
    && git fetch --all \
-    # commit: from branch sombernight/electrum_20210421
-    && git checkout "5356bc7838b03c8c174c91fe01539c91d1b40b9f^{commit}" \
-    && python3 -m pip install --user -e .
+    # commit: from branch sombernight/electrum_20210421b
+    && git checkout "cdee188f0ef28ff8452207da409912da19e917ca^{commit}" \
+    && python3 -m pip install --no-dependencies --user -e .

 # build env vars
 ENV USE_SDK_WRAPPER=1
--- a/contrib/android/Makefile
+++ b/contrib/android/Makefile
@@ -1,5 +1,14 @@
+SHELL := /bin/bash
 PYTHON = python3

+# for reproducible builds
+export LC_ALL             := C
+export TZ                 := UTC
+export SOURCE_DATE_EPOCH  := $(shell git log -1 --pretty=%ct)
+export PYTHONHASHSEED     := $(SOURCE_DATE_EPOCH)
+export BUILD_DATE         := $(shell LC_ALL=C TZ=UTC date +'%b %e %Y' -d @$(SOURCE_DATE_EPOCH))
+export BUILD_TIME         := $(shell LC_ALL=C TZ=UTC date +'%H:%M:%S' -d @$(SOURCE_DATE_EPOCH))
+
 # needs kivy installed or in PYTHONPATH

 .PHONY: theming apk clean
--- a/contrib/android/Readme.md
+++ b/contrib/android/Readme.md
@@ -5,8 +5,8 @@ To generate an APK file, follow these instructions.

 ## Android binary with Docker

-✗ _This script does not produce reproducible output (yet!).
-   Please help us remedy this._
+✓ _These binaries should be reproducible, meaning you should be able to generate
+   binaries that match the official releases._

 This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
 similar system. The docker commands should be executed in the project's root
@@ -24,24 +24,27 @@ folder.
 2. Build image

    ```
-    $ sudo docker build -t electrum-android-builder-img contrib/android
+    $ ./contrib/android/build_docker_image.sh
    ```

-3. Build locale files
+3. Build binaries
+
+    It's recommended to build from a fresh clone
+    (but you can skip this if reproducibility is not necessary).

    ```
-    $ ./contrib/pull_locale
+    $ FRESH_CLONE=contrib/android/fresh_clone && \
+        sudo rm -rf $FRESH_CLONE && \
+        umask 0022 && \
+        mkdir -p $FRESH_CLONE && \
+        cd $FRESH_CLONE  && \
+        git clone https://github.com/spesmilo/electrum.git && \
+        cd electrum
    ```

-4. Prepare pure python dependencies
-
-    ```
-    $ ./contrib/make_packages
-    ```
-
-5. Build binaries
-
+    And then build from this directory:
    ```
+    $ git checkout $REV
    $ mkdir --parents $PWD/.buildozer/.gradle
    $ sudo docker run -it --rm \
        --name electrum-android-builder-cont \
@@ -52,6 +55,7 @@ folder.
        electrum-android-builder-img \
        ./contrib/android/make_apk
    ```
+    
    This mounts the project dir inside the container,
    and so the modifications will affect it, e.g. `.buildozer` folder
    will be created.
@@ -125,3 +129,22 @@ $ adb shell
 $ run-as org.electrum.electrum ls /data/data/org.electrum.electrum/files/data
 $ run-as org.electrum.electrum cp /data/data/org.electrum.electrum/files/data/wallets/my_wallet /sdcard/some_path/my_wallet
 ```
+
+### How to investigate diff between binaries if reproducibility fails?
+```
+cd bin/
+unzip Electrum-*.apk1 -d apk1
+mkdir apk1/assets/private_mp3/
+tar -xzvf apk1/assets/private.mp3 --directory apk1/assets/private_mp3/
+
+unzip Electrum-*.apk2 -d apk2
+mkdir apk2/assets/private_mp3/
+tar -xzvf apk2/assets/private.mp3 --directory apk2/assets/private_mp3/
+
+sudo chown --recursive "$(id -u -n)" apk1/ apk2/
+chmod -R +Xr  apk1/ apk2/
+$(cd apk1; find -type f -exec sha256sum '{}' \; > ./../sha256sum1)
+$(cd apk2; find -type f -exec sha256sum '{}' \; > ./../sha256sum2)
+diff sha256sum1 sha256sum2 > d
+cat d
+```
--- a/contrib/android/build_docker_image.sh
+++ b/contrib/android/build_docker_image.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+CONTRIB_ANDROID="$(dirname "$(readlink -e "$0")")"
+CONTRIB="$CONTRIB_ANDROID"/..
+
+cp "$CONTRIB/deterministic-build/requirements-build-android.txt" "$CONTRIB_ANDROID/requirements-build-android.txt"
+sudo docker build -t electrum-android-builder-img "$CONTRIB_ANDROID"
+rm "$CONTRIB_ANDROID/requirements-build-android.txt"
--- a/contrib/android/buildozer.spec
+++ b/contrib/android/buildozer.spec
@@ -26,7 +26,9 @@ source.exclude_dirs = bin, build, dist, contrib,
    packages/qdarkstyle,
    packages/qtpy
 # (list) List of exclusions using pattern matching
-source.exclude_patterns = Makefile,setup*
+source.exclude_patterns = Makefile,setup*,
+    # not reproducible:
+    packages/aiohttp-*.dist-info/*

 # (str) Application versioning (method 1)
 version.regex = APK_VERSION = '(.*)'
@@ -79,7 +81,7 @@ android.api = 29
 android.minapi = 21

 # (str) Android NDK version to use
-android.ndk = 19c
+android.ndk = 22b

 # (int) Android NDK API to use (optional). This is the minimum API your app will support.
 android.ndk_api = 21
@@ -96,6 +98,18 @@ android.sdk_path = /opt/android/android-sdk
 # (str) ANT directory (if empty, it will be automatically downloaded.)
 android.ant_path = /opt/android/apache-ant

+# (bool) If True, then skip trying to update the Android sdk
+# This can be useful to avoid excess Internet downloads or save time
+# when an update is due and you just want to test/build your package
+# note(ghost43): probably needed for reproducibility. versions pinned in Dockerfile.
+android.skip_update = True
+
+# (bool) If True, then automatically accept SDK license
+# agreements. This is intended for automation only. If set to False,
+# the default, you will be shown the license when first running
+# buildozer.
+android.accept_sdk_license = True
+
 # (str) Android entry point, default is ok for Kivy-based app
 #android.entrypoint = org.renpy.android.PythonActivity

--- a/contrib/android/make_apk
+++ b/contrib/android/make_apk
@@ -3,25 +3,42 @@
 set -e

 CONTRIB_ANDROID="$(dirname "$(readlink -e "$0")")"
-ROOT_FOLDER="$CONTRIB_ANDROID"/../..
-PACKAGES="$ROOT_FOLDER"/packages/
-LOCALE="$ROOT_FOLDER"/electrum/locale/
+CONTRIB="$CONTRIB_ANDROID"/..
+PROJECT_ROOT="$CONTRIB"/..
+PACKAGES="$PROJECT_ROOT"/packages/
+LOCALE="$PROJECT_ROOT"/electrum/locale/

-if [ ! -d "$LOCALE" ]; then
-  echo "Run pull_locale first!"
-  exit 1
-fi
+. "$CONTRIB"/build_tools_util.sh

 if [ ! -d "$PACKAGES" ]; then
-  echo "Run make_packages first!"
-  exit 1
+  "$CONTRIB"/make_packages || fail "make_packages failed"
 fi

-pushd ./contrib/android
+pushd "$PROJECT_ROOT"
+git submodule update --init
+popd

-make theming
+# update locale
+info "preparing electrum-locale."
+(
+    cd "$CONTRIB"/deterministic-build/electrum-locale
+    if ! which msgfmt > /dev/null 2>&1; then
+        fail "Please install gettext"
+    fi
+    # we want the binary to have only compiled (.mo) locale files; not source (.po) files
+    rm -rf "$PROJECT_ROOT/electrum/locale/"
+    for i in ./locale/*; do
+        dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES"
+        mkdir -p $dir
+        msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true
+    done
+)

+pushd "$CONTRIB_ANDROID"
+
+info "apk building phase starts."
 if [[ -n "$1"  && "$1" == "release" ]] ; then
+    # do release build, and sign the APKs.
    echo -n Keystore Password:
    read -s password
    export P4A_RELEASE_KEYSTORE=~/.keystore
@@ -33,7 +50,15 @@ if [[ -n "$1"  && "$1" == "release" ]] ; then
    make release
    export APP_ANDROID_ARCH=arm64-v8a
    make release
+elif [[ -n "$1"  && "$1" == "release-unsigned" ]] ; then
+    # do release build, but do not sign the APKs.
+    # build two apks
+    export APP_ANDROID_ARCH=armeabi-v7a
+    make release
+    export APP_ANDROID_ARCH=arm64-v8a
+    make release
 else
+    # do debug build; the default.
    export P4A_DEBUG_KEYSTORE="$CONTRIB_ANDROID"/android_debug.keystore
    export P4A_DEBUG_KEYSTORE_PASSWD=unsafepassword
    export P4A_DEBUG_KEYALIAS_PASSWD=unsafepassword