Merge pull request #8167 from SomberNight/202301_release

release process: split release.sh into two

contrib/add_cosigner

@@ -3,12 +3,15 @@
 # This script is part of the workflow for BUILDERs to reproduce and sign the
 # release binaries. (for builders who do not have sftp access to "electrum-downloads-airlock")
 #
+# env vars:
+# - SSHUSER
+#
+#
 # - BUILDER builds all binaries and checks they match the official releases
 #   (using release.sh, and perhaps some manual steps)
 # - BUILDER creates a PR against https://github.com/spesmilo/electrum-signatures/
 #   to add their sigs for a given release, which then gets merged
-# - SFTPUSER runs `$ electrum/contrib/add_cosigner $BUILDER`
-# - SFTPUSER runs `$ SSHUSER=$SFTPUSER electrum/contrib/upload.sh`
+# - SFTPUSER runs `$ SSHUSER=$SFTPUSER electrum/contrib/add_cosigner $BUILDER`
 # - SFTPUSER runs `$ electrum/contrib/make_download $WWW_DIR`
 # -     $ (cd $WWW_DIR; git commit -a -m "add_cosigner"; git push)
 # - SFTPUSER runs `$ electrum-web/publish.sh $SFTPUSER`
@@ -18,6 +21,7 @@ import re
 import os
 import sys
 import importlib
+import subprocess
 
 
 # cd to project root
@@ -32,7 +36,7 @@ ELECTRUM_VERSION = version_module.ELECTRUM_VERSION
 APK_VERSION = version_module.APK_VERSION
 print("version", ELECTRUM_VERSION)
 
-# GPG names of cosigner
+# GPG name of cosigner
 cosigner = sys.argv[1]
 
 version = version_win = version_mac = ELECTRUM_VERSION
@@ -63,3 +67,6 @@ for shortname, filename in files.items():
     os.system(f"wget -nc {sig_url} -O {sig_path}")
     if os.system(f"gpg --verify {sig_path} {path}") != 0:
         raise Exception(sig_name)
+
+print("Calling upload.sh now... This might take some time.")
+subprocess.check_output(["./contrib/upload.sh", ])

contrib/release.sh

@@ -1,23 +1,19 @@
 #!/bin/bash
 #
-# This script, for the RELEASEMANAGER:
-# - builds and uploads all binaries,
+# This script is used for stage 1 of the release process. It operates exclusively on the airlock.
+# This script, for the RELEASEMANAGER (RM):
+# - builds and uploads all binaries to airlock,
 # - assumes all keys are available, and signs everything
 # This script, for other builders:
 # - builds all reproducible binaries,
-# - downloads binaries built by the release manager, compares and signs them,
+# - downloads binaries built by the release manager (from airlock), compares and signs them,
 # - and then uploads sigs
 # Note: the .dmg should be built separately beforehand and copied into dist/
 #       (as it is built on a separate machine)
 #
+#
 # env vars:
 # - ELECBUILD_NOCACHE: if set, forces rebuild of docker images
-# - WWW_DIR: path to "electrum-web" git clone
-#
-# additional env vars for the RELEASEMANAGER:
-# - for signing the version announcement file:
-#   - ELECTRUM_SIGNING_ADDRESS (required)
-#   - ELECTRUM_SIGNING_WALLET (required)
 #
 # "uploadserver" is set in /etc/hosts
 #
@@ -29,6 +25,20 @@
 # - update RELEASE-NOTES and version.py
 # - $ git tag -s $VERSION -m $VERSION
 #
+# -----
+# Then, typical release flow:
+# - RM runs release.sh
+# - Another SFTPUSER BUILDER runs `$ ./release.sh`
+# - now airlock contains new binaries and two sigs for each
+# - deploy.sh will verify sigs and move binaries across airlock
+# - new binaries are now publicly available on uploadserver, but not linked from website yet
+# - other BUILDERS can now also try to reproduce binaries and open PRs with sigs against spesmilo/electrum-signatures
+#   - these PRs can get merged as they come
+#   - run add_cosigner
+# - after some time, RM can run release_www.sh to create and commit website-update
+#   - then run WWW_DIR/publish.sh to update website
+# - at least two people need to run WWW_DIR/publish.sh
+#
 
 set -e
 
@@ -42,10 +52,6 @@ cd "$PROJECT_ROOT"
 # rm -rf dist/*
 # rm -f .buildozer
 
-if [ -z "$WWW_DIR" ] ; then
-    WWW_DIR=/opt/electrum-web
-fi
-
 GPGUSER=$1
 if [ -z "$GPGUSER" ]; then
     fail "usage: $0 gpg_username"
@@ -247,13 +253,6 @@ else
 
     cd "$PROJECT_ROOT"
 
-    info "updating www repo"
-    ./contrib/make_download $WWW_DIR
-    info "signing the version announcement file"
-    sig=$(./run_electrum -o signmessage $ELECTRUM_SIGNING_ADDRESS $VERSION -w $ELECTRUM_SIGNING_WALLET)
-    echo "{ \"version\":\"$VERSION\", \"signatures\":{ \"$ELECTRUM_SIGNING_ADDRESS\":\"$sig\"}}" > $WWW_DIR/version
-
-
     if [ $REV != $VERSION ]; then
         fail "versions differ, not uploading"
     fi
@@ -266,14 +265,10 @@ else
         touch dist/uploaded
     fi
 
-    # push changes to website repo
-    pushd $WWW_DIR
-    git diff
-    git commit -a -m "version $VERSION"
-    git push
-    popd
 fi
 
 
 info "release.sh finished successfully."
-info "now you should run WWW_DIR/publish.sh to sign the website commit and upload signature"
+info "After two people ran release.sh, the binaries will be publicly available on uploadserver."
+info "Then, we wait for additional signers, and run add_cosigner for them."
+info "Finally, release_www.sh needs to be run, for the website to be updated."

contrib/release_www.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# env vars:
+# - WWW_DIR: path to "electrum-web" git clone
+# - for signing the version announcement file:
+#   - ELECTRUM_SIGNING_ADDRESS (required)
+#   - ELECTRUM_SIGNING_WALLET (required)
+#
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/.."
+CONTRIB="$PROJECT_ROOT/contrib"
+
+cd "$PROJECT_ROOT"
+
+. "$CONTRIB"/build_tools_util.sh
+
+
+echo -n "Remember to run add_cosigner to add any additional sigs.  Continue (y/n)? "
+read answer
+if [ "$answer" != "y" ]; then
+    echo "exit"
+    exit 1
+fi
+
+
+if [ -z "$WWW_DIR" ] ; then
+    WWW_DIR=/opt/electrum-web
+fi
+
+if [ -z "$ELECTRUM_SIGNING_WALLET" ] || [ -z "$ELECTRUM_SIGNING_ADDRESS" ]; then
+    echo "You need to set env vars ELECTRUM_SIGNING_WALLET and ELECTRUM_SIGNING_ADDRESS!"
+    exit 1
+fi
+
+VERSION=$(python3 -c "import electrum; print(electrum.version.ELECTRUM_VERSION)")
+info "VERSION: $VERSION"
+
+set -x
+
+info "updating www repo"
+./contrib/make_download "$WWW_DIR"
+info "signing the version announcement file"
+sig=$(./run_electrum -o signmessage $ELECTRUM_SIGNING_ADDRESS $VERSION -w $ELECTRUM_SIGNING_WALLET)
+echo "{ \"version\":\"$VERSION\", \"signatures\":{ \"$ELECTRUM_SIGNING_ADDRESS\":\"$sig\"}}" > "$WWW_DIR"/version
+
+# push changes to website repo
+pushd "$WWW_DIR"
+git diff
+git commit -a -m "version $VERSION"
+git push
+popd
+
+
+info "release_www.sh finished successfully."
+info "now you should run WWW_DIR/publish.sh to sign the website commit and upload signature"

contrib/trigger_deploy.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Triggers deploy.sh to maybe update the website or move binaries.
+# uploadserver needs to be defined in /etc/hosts
+
+SSHUSER=$1
+TRIGGERVERSION=$2
+if [ -z $SSHUSER ] || [ -z TRIGGERVERSION ]; then
+    echo "usage: $0 SSHUSER TRIGGERVERSION"
+    echo "e.g. $0 thomasv 3.0.0"
+    echo "e.g. $0 thomasv website"
+    exit 1
+fi
+set -ex
+cd "$(dirname "$0")"
+
+if [ "$TRIGGERVERSION" == "website" ]; then
+    rm -f trigger_website
+    touch trigger_website
+    echo "uploading file: trigger_website..."
+    sftp -oBatchMode=no -b - "$SSHUSER@uploadserver" << !
+       cd electrum-downloads-airlock
+       mput trigger_website
+       bye
+!
+else
+    rm -f trigger_binaries
+    printf "$TRIGGERVERSION" > trigger_binaries
+    echo "uploading file: trigger_binaries..."
+    sftp -oBatchMode=no -b - "$SSHUSER@uploadserver" << !
+       cd electrum-downloads-airlock
+       mput trigger_binaries
+       bye
+!
+fi
+

contrib/upload.sh

@@ -5,9 +5,10 @@
 # - ELECBUILD_UPLOADFROM
 # - SSHUSER
 
-set -e
+set -ex
 
 PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/.."
+CONTRIB="$PROJECT_ROOT/contrib"
 
 if [ -z "$SSHUSER" ]; then
     SSHUSER=thomasv
@@ -15,8 +16,8 @@ fi
 
 cd "$PROJECT_ROOT"
 
-version=$(git describe --tags --abbrev=0)
-echo $version
+VERSION=$(python3 -c "import electrum; print(electrum.version.ELECTRUM_VERSION)")
+echo "$VERSION"
 
 if [ -z "$ELECBUILD_UPLOADFROM" ]; then
     cd "$PROJECT_ROOT/dist"
@@ -30,9 +31,12 @@ fi
 
 sftp -oBatchMode=no -b - "$SSHUSER@uploadserver" << !
    cd electrum-downloads-airlock
-   -mkdir "$version"
-   -chmod 777 "$version"
-   cd "$version"
-   mput *
+   -mkdir "$VERSION"
+   -chmod 777 "$VERSION"
+   cd "$VERSION"
+   -mput *
+   -chmod 444 *  # this prevents future re-uploads of same file
    bye
 !
+
+"$CONTRIB/trigger_deploy.sh" "$SSHUSER" "$VERSION"