build: android reprod: "pip install" needs "--no-build-isolation"

maybe fixes https://github.com/spesmilo/electrum/issues/7640

Looks like by default pip is ignoring the locally available setuptools and wheel,
and downloading the latest ones from the internet at build time...

https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/?highlight=no-build-isolation#disabling-build-isolation
https://stackoverflow.com/a/62889268

> When making build requirements available, pip does so in an isolated environment. That is, pip does not install those requirements into the user’s site-packages, but rather installs them in a temporary directory which it adds to the user’s sys.path for the duration of the build. This ensures that build requirements are handled independently of the user’s runtime environment. For example, a project that needs a recent version of setuptools to build can still be installed, even if the user has an older version installed (and without silently replacing that version).
>
> In certain cases, projects (or redistributors) may have workflows that explicitly manage the build environment. For such workflows, build isolation can be problematic. If this is the case, pip provides a --no-build-isolation flag to disable build isolation. Users supplying this flag are responsible for ensuring the build environment is managed appropriately (including ensuring that all required build dependencies are installed).

If only it were that easy!
If we add the "--no-build-isolation" flag, it becomes our responsibility to install *all* build time deps,
hence we now have "requirements-build-makepackages.txt".

.gitignore

@@ -44,6 +44,7 @@ contrib/android/android_debug.keystore
 contrib/secp256k1/
 contrib/zbar/
 contrib/libusb/
+contrib/.venv_make_packages/
 
 # shared objects
 electrum/*.so

contrib/android/Dockerfile

@@ -108,6 +108,7 @@ RUN apt -y update -qq \
         python3-dev \
         python3-pip \
         python3-setuptools \
+        python3-venv \
         wget \
         lbzip2 \
         patch \

contrib/build-linux/sdist/Dockerfile

@@ -10,6 +10,7 @@ RUN apt-get update -q && \
         python3 \
         python3-pip \
         python3-setuptools \
+        python3-venv \
         faketime \
         && \
     rm -rf /var/lib/apt/lists/* && \

contrib/build-linux/sdist/make_sdist.sh

@@ -16,13 +16,9 @@ python3 --version || fail "python interpreter not found"
 break_legacy_easy_install
 
 # upgrade to modern pip so that it knows the flags we need.
-# we will then install a pinned version of pip as part of requirements-build-sdist
+# (make_packages will later install a pinned version of pip in a venv)
 python3 -m pip install --upgrade pip
 
-info "Installing pinned requirements."
-python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-sdist.txt
-
-
 "$CONTRIB"/make_packages || fail "make_packages failed"
 
 git submodule update --init

contrib/deterministic-build/requirements-build-makepackages.txt

@@ -0,0 +1,21 @@
+packaging==21.3 \
+    --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \
+    --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522
+pip==21.3.1 \
+    --hash=sha256:deaf32dcd9ab821e359cd8330786bcd077604b5c5730c0b096eda46f95c24a2d \
+    --hash=sha256:fd11ba3d0fdb4c07fbc5ecbba0b1b719809420f25038f8ee3cd913d3faa3033a
+pyparsing==3.0.7 \
+    --hash=sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea \
+    --hash=sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484
+setuptools==59.6.0 \
+    --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \
+    --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e
+setuptools-scm==6.4.2 \
+    --hash=sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30 \
+    --hash=sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4
+tomli==1.2.3 \
+    --hash=sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f \
+    --hash=sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c
+wheel==0.37.1 \
+    --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \
+    --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4

contrib/deterministic-build/requirements-build-sdist.txt

@@ -1,9 +0,0 @@
-pip==21.3.1 \
-    --hash=sha256:deaf32dcd9ab821e359cd8330786bcd077604b5c5730c0b096eda46f95c24a2d \
-    --hash=sha256:fd11ba3d0fdb4c07fbc5ecbba0b1b719809420f25038f8ee3cd913d3faa3033a
-setuptools==59.6.0 \
-    --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \
-    --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e
-wheel==0.37.1 \
-    --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \
-    --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4

contrib/freeze_packages.sh

@@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit
 
 ${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
 
-for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-sdist' '-build-appimage' '-build-android'; do
+for i in '' '-hw' '-binaries' '-binaries-mac' '-build-wine' '-build-mac' '-build-makepackages' '-build-appimage' '-build-android'; do
     rm -rf "$venv_dir"
     virtualenv -p ${SYSTEM_PYTHON} $venv_dir
 

contrib/make_packages

@@ -13,6 +13,16 @@ if [ -d "$PACKAGES" ]; then
   rm -r "$PACKAGES"
 fi
 
+# create virtualenv
+# note: venv path needs to be deterministic as some produced files will contain it
+venv_dir="$CONTRIB/.venv_make_packages/"
+rm -rf "$venv_dir"
+python3 -m venv "$venv_dir"
+source "$venv_dir"/bin/activate
+
+# installing pinned build-time requirements, such as pip/wheel/setuptools
+python -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-makepackages.txt
+
 # opt out of compiling C extensions
 # FIXME aiohttp opt-out is not released yet: https://github.com/aio-libs/aiohttp/pull/3828
 export AIOHTTP_NO_EXTENSIONS=1
@@ -34,6 +44,7 @@ export BUILD_TIME="$(LC_ALL=C TZ=UTC date +'%H:%M:%S' -d @$SOURCE_DATE_EPOCH)"
 #       Hence "aiohttp-*.dist-info/" is not reproducible either.
 #       All this means that downstream users of this script, such as the sdist build
 #       and the android apk build need to make sure these files get excluded.
-python3 -m pip install --no-compile --no-dependencies --no-binary :all: \
+# note: --no-build-isolation is needed so that pip uses the locally available setuptools and wheel,
+#       instead of downloading the latest ones
+python3 -m pip install --no-build-isolation --no-compile --no-dependencies --no-binary :all: \
     -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
-

contrib/requirements/requirements-build-makepackages.txt

@@ -0,0 +1,14 @@
+# This file contains build-time dependencies needed to be able to build our pure python dependencies.
+# For reproducibility, some build-time deps, most notably "wheel", need to be pinned. (see #7640)
+# By default, when doing e.g. "pip install", pip downloads the latest version of wheel (and setuptools, etc),
+# regardless whether a sufficiently recent version of wheel is already installed locally...
+# The only way I have found to avoid this, is to use the "--no-build-isolation" flag,
+# in which case it becomes our responsibility to install *all* build time deps...
+
+pip
+setuptools
+wheel
+
+# importlib_metadata also needs:
+# https://github.com/python/importlib_metadata/blob/1e2381fe101fd70742a0171e51c1be82aedf519b/pyproject.toml#L2
+setuptools_scm[toml]>=3.4.1

contrib/requirements/requirements-build-sdist.txt

@@ -1,4 +0,0 @@
-# need modern versions of pip (and maybe other build tools), the one in apt had issues
-pip
-setuptools
-wheel