build: also build a "source-only" linux-packager-friendly tarball

We now distribute two tarballs, a "normal" one (the default, recommended for users), and a new strictly source-only one (for Linux distro packagers). The normal tarball, in addition to including everything from the source-only one, also includes: - compiled (`.mo`) locale files (in addition to source `.po` locale files) - compiled (`_pb2.py`) protobuf files (in addition to source `.proto` files) - the `packages/` folder containing source-only pure-python runtime dependencies merges https://github.com/spesmilo/electrum/pull/7594 with follow-ups
2022-08-16 17:25:24 +00:00
parent 4dee6e53b9 7d36f2ba3a
commit d0de44a794
9 changed files with 111 additions and 36 deletions
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -208,7 +208,6 @@ task:
    path: "dist/*"

 task:
-  name: tarball build
  container:
    dockerfile: contrib/build-linux/sdist/Dockerfile
    cpu: 1
@@ -217,6 +216,11 @@ task:
    - ./contrib/build-linux/sdist/make_sdist.sh
  binaries_artifacts:
    path: "dist/*"
+  matrix:
+    - name: tarball build
+    - name: source-only tarball build
+      env:
+        OMIT_UNCLEAN_FILES: 1

 task:
  name: Submodules
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -8,8 +8,7 @@ recursive-include packages cacert.pem

 include contrib/requirements/requirements*.txt
 include contrib/deterministic-build/requirements*.txt
-include contrib/make_libsecp256k1.sh
-include contrib/build_tools_util.sh
+include contrib/*.sh

 graft electrum
 prune electrum/tests
--- a/contrib/build-linux/sdist/README.md
+++ b/contrib/build-linux/sdist/README.md
@@ -1,5 +1,4 @@
-Source tarballs
-===============
+# Source tarballs

 ✓ _This file should be reproducible, meaning you should be able to generate
   distributables that match the official releases._
@@ -7,18 +6,31 @@ Source tarballs
 This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
 similar system.

+We distribute two tarballs, a "normal" one (the default, recommended for users),
+and a strictly source-only one (for Linux distro packagers).
+The normal tarball, in addition to including everything from
+the source-only one, also includes:
+- compiled (`.mo`) locale files (in addition to source `.po` locale files)
+- compiled (`_pb2.py`) protobuf files (in addition to source `.proto` files)
+- the `packages/` folder containing source-only pure-python runtime dependencies
+
+
+## Build steps
+
 1. Install Docker

    See `contrib/docker_notes.md`.

-2. Build source tarball
+2. Build tarball

+    (set envvar `OMIT_UNCLEAN_FILES=1` to build the "source-only" tarball)
    ```
    $ ./build.sh
    ```
    If you want reproducibility, try instead e.g.:
    ```
    $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./build.sh
+    $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 OMIT_UNCLEAN_FILES=1 ./build.sh
    ```

 3. The generated distributables are in `./dist`.
--- a/contrib/build-linux/sdist/build.sh
+++ b/contrib/build-linux/sdist/build.sh
@@ -47,6 +47,7 @@ docker run -it \
    -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrum \
    --rm \
    --workdir /opt/electrum/contrib/build-linux/sdist \
+    --env OMIT_UNCLEAN_FILES \
    electrum-sdist-builder-img \
    ./make_sdist.sh

--- a/contrib/build-linux/sdist/make_sdist.sh
+++ b/contrib/build-linux/sdist/make_sdist.sh
@@ -6,7 +6,7 @@ PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
 CONTRIB="$PROJECT_ROOT/contrib"
 CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
 DISTDIR="$PROJECT_ROOT/dist"
-LOCALE="$PROJECT_ROOT/electrum/locale/"
+LOCALE="$PROJECT_ROOT/electrum/locale"

 . "$CONTRIB"/build_tools_util.sh

@@ -19,35 +19,55 @@ break_legacy_easy_install
 # (make_packages will later install a pinned version of pip in a venv)
 python3 -m pip install --upgrade pip

-"$CONTRIB"/make_packages || fail "make_packages failed"
+rm -rf "$PROJECT_ROOT/packages/"
+if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then
+  "$CONTRIB"/make_packages || fail "make_packages failed"
+fi

 git submodule update --init

 (
-    cd "$CONTRIB/deterministic-build/electrum-locale/"
-    if ! which msgfmt > /dev/null 2>&1; then
-        echo "Please install gettext"
-        exit 1
-    fi
-    # We include both source (.po) and compiled (.mo) locale files in the source dist.
-    # Maybe we should exclude the compiled locale files? see https://askubuntu.com/a/144139
-    # (also see MANIFEST.in)
+    # By default, include both source (.po) and compiled (.mo) locale files in the source dist.
+    # Set option OMIT_UNCLEAN_FILES=1 to exclude the compiled locale files
+    # see https://askubuntu.com/a/144139 (also see MANIFEST.in)
    rm -rf "$LOCALE"
-    for i in ./locale/*; do
-        dir="$PROJECT_ROOT/electrum/$i/LC_MESSAGES"
-        mkdir -p "$dir"
-        msgfmt --output-file="$dir/electrum.mo" "$i/electrum.po" || true
-        cp $i/electrum.po "$PROJECT_ROOT/electrum/$i/electrum.po"
-    done
+    cp -r "$CONTRIB/deterministic-build/electrum-locale/locale/" "$LOCALE/"
+    if ([ "$OMIT_UNCLEAN_FILES" != 1 ]); then
+      "$CONTRIB/build_locale.sh" "$LOCALE"
+    fi
 )

+if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then
+    # FIXME side-effecting repo... though in practice, this script probably runs in fresh_clone
+    rm -f "$PROJECT_ROOT/electrum/paymentrequest_pb2.py"
+fi
+
 (
    cd "$PROJECT_ROOT"

    find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +

    # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963
-    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar
+    if ([ "$OMIT_UNCLEAN_FILES" = 1 ])
+        then PY_DISTDIR="dist/_sourceonly" # The DISTDIR variable of this script is only used to find where the output is *finally* placed.
+        else PY_DISTDIR="dist"
+    fi
+    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar --dist-dir="$PY_DISTDIR"
+    if ([ "$OMIT_UNCLEAN_FILES" = 1 ]); then
+        python3 <<EOF
+import importlib.util
+import os
+
+# load version.py; needlessly complicated alternative to "imp.load_source":
+version_spec = importlib.util.spec_from_file_location('version', 'electrum/version.py')
+version_module = importlib.util.module_from_spec(version_spec)
+version_spec.loader.exec_module(version_module)
+
+VER = version_module.ELECTRUM_VERSION
+os.rename(f"dist/_sourceonly/Electrum-{VER}.tar.gz", f"dist/Electrum-sourceonly-{VER}.tar.gz")
+EOF
+        rmdir "$PY_DISTDIR"
+    fi
 )


--- a/contrib/build_locale.sh
+++ b/contrib/build_locale.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if [ ! -d "$1" ]; then
+    echo "usage: $0 path/to/locale"
+    exit 1
+fi
+
+if ! which msgfmt > /dev/null 2>&1; then
+    echo "Please install gettext"
+    exit 1
+fi
+
+for i in "$1/"*; do
+  mkdir -p "$i/LC_MESSAGES"
+  (msgfmt --output-file="$i/LC_MESSAGES/electrum.mo" "$i/electrum.po" || true)
+done
--- a/contrib/generate_payreqpb2.sh
+++ b/contrib/generate_payreqpb2.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Generates the file paymentrequest_pb2.py
+
+CONTRIB="$(dirname "$(readlink -e "$0")")"
+EL="$CONTRIB"/../electrum
+
+if ! which protoc > /dev/null 2>&1; then
+    echo "Please install 'protoc'"
+    echo "If you're on Debian, try 'sudo apt install protobuf-compiler'?"
+    exit 1
+fi
+
+protoc --proto_path="$EL" --python_out="$EL" "$EL"/paymentrequest.proto
--- a/contrib/release.sh
+++ b/contrib/release.sh
@@ -88,6 +88,14 @@ else
   ./contrib/build-linux/sdist/build.sh
 fi

+# create source-only tarball
+srctarball="Electrum-sourceonly-$VERSION.tar.gz"
+if test -f "dist/$srctarball"; then
+    info "file exists: $srctarball"
+else
+   OMIT_UNCLEAN_FILES=1 ./contrib/build-linux/sdist/build.sh
+fi
+
 # appimage
 appimage="electrum-$REV-x86_64.AppImage"
 if test -f "dist/$appimage"; then
@@ -186,15 +194,17 @@ if [ -z "$RELEASEMANAGER" ] ; then
       bye
 !
    # check we have each binary
-    test -f "$tarball"  || fail "tarball not found among sftp downloads"
-    test -f "$appimage" || fail "appimage not found among sftp downloads"
-    test -f "$win1"     || fail "win1 not found among sftp downloads"
-    test -f "$win2"     || fail "win2 not found among sftp downloads"
-    test -f "$win3"     || fail "win3 not found among sftp downloads"
-    test -f "$apk1"     || fail "apk1 not found among sftp downloads"
-    test -f "$apk2"     || fail "apk2 not found among sftp downloads"
-    test -f "$dmg"      || fail "dmg not found among sftp downloads"
+    test -f "$tarball"    || fail "tarball not found among sftp downloads"
+    test -f "$srctarball" || fail "srctarball not found among sftp downloads"
+    test -f "$appimage"   || fail "appimage not found among sftp downloads"
+    test -f "$win1"       || fail "win1 not found among sftp downloads"
+    test -f "$win2"       || fail "win2 not found among sftp downloads"
+    test -f "$win3"       || fail "win3 not found among sftp downloads"
+    test -f "$apk1"       || fail "apk1 not found among sftp downloads"
+    test -f "$apk2"       || fail "apk2 not found among sftp downloads"
+    test -f "$dmg"        || fail "dmg not found among sftp downloads"
    test -f "$PROJECT_ROOT/dist/$tarball"    || fail "tarball not found among built files"
+    test -f "$PROJECT_ROOT/dist/$srctarball" || fail "srctarball not found among built files"
    test -f "$PROJECT_ROOT/dist/$appimage"   || fail "appimage not found among built files"
    test -f "$CONTRIB/build-wine/dist/$win1" || fail "win1 not found among built files"
    test -f "$CONTRIB/build-wine/dist/$win2" || fail "win2 not found among built files"
@@ -203,8 +213,9 @@ if [ -z "$RELEASEMANAGER" ] ; then
    test -f "$PROJECT_ROOT/dist/$apk2"       || fail "apk2 not found among built files"
    test -f "$PROJECT_ROOT/dist/$dmg"        || fail "dmg not found among built files"
    # compare downloaded binaries against ones we built
-    cmp --silent "$tarball" "$PROJECT_ROOT/dist/$tarball" || fail "files are different. tarball."
-    cmp --silent "$appimage" "$PROJECT_ROOT/dist/$appimage" || fail "files are different. appimage."
+    cmp --silent "$tarball"    "$PROJECT_ROOT/dist/$tarball"    || fail "files are different. tarball."
+    cmp --silent "$srctarball" "$PROJECT_ROOT/dist/$srctarball" || fail "files are different. srctarball."
+    cmp --silent "$appimage"   "$PROJECT_ROOT/dist/$appimage"   || fail "files are different. appimage."
    rm -rf "$CONTRIB/build-wine/signed/" && mkdir --parents "$CONTRIB/build-wine/signed/"
    cp -f "$win1" "$win2" "$win3" "$CONTRIB/build-wine/signed/"
    "$CONTRIB/build-wine/unsign.sh" || fail "files are different. windows."
@@ -214,7 +225,7 @@ if [ -z "$RELEASEMANAGER" ] ; then
    # all files matched. sign them.
    rm -rf "$PROJECT_ROOT/dist/sigs/"
    mkdir --parents "$PROJECT_ROOT/dist/sigs/"
-    for fname in "$tarball" "$appimage" "$win1" "$win2" "$win3" "$apk1" "$apk2" "$dmg" ; do
+    for fname in "$tarball" "$srctarball" "$appimage" "$win1" "$win2" "$win3" "$apk1" "$apk2" "$dmg" ; do
        signame="$fname.$GPGUSER.asc"
        gpg --sign --armor --detach $PUBKEY --output "$PROJECT_ROOT/dist/sigs/$signame" "$fname"
    done
--- a/electrum/paymentrequest.py
+++ b/electrum/paymentrequest.py
@@ -36,8 +36,7 @@ import aiohttp
 try:
    from . import paymentrequest_pb2 as pb2
 except ImportError:
-    # sudo apt-get install protobuf-compiler
-    sys.exit("Error: could not find paymentrequest_pb2.py. Create it with 'protoc --proto_path=electrum/ --python_out=electrum/ electrum/paymentrequest.proto'")
+    sys.exit("Error: could not find paymentrequest_pb2.py. Create it with 'contrib/generate_payreqpb2.sh'")

 from . import bitcoin, constants, ecc, util, transaction, x509, rsakey
 from .util import bh2u, bfh, make_aiohttp_session