move files to FastSync

2018-11-30 12:54:16 +09:00
parent c6f1f46800
commit b5e47f94cc
5 changed files with 0 additions and 0 deletions
--- a/contrib/FastSync/NicolasDorier.utxo-sets.asc
+++ b/contrib/FastSync/NicolasDorier.utxo-sets.asc
@@ -0,0 +1,21 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA256
+
+fab994299273080bf7124c8c45c4ada867974ca747900178496a69e450cf713f  utxo-snapshot-bitcoin-mainnet-551636.tar
+eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742  utxo-snapshot-bitcoin-testnet-1445586.tar
+-----BEGIN PGP SIGNATURE-----
+
+wsFcBAEBCAAQBQJcALGMCRBmGHY+8JGG/gAA0YsQAJgfTLHy/z+5c7pumb7QGl5y
+afs7KrDUpc5jM6c6jeaKaysWm/aQmWXYtTYMFeN2+vexVLvbMiYSVMl5ts2zRScl
+tY+HQ2aYlUP754S98gjFrWNner5wpREhu+UILtzB79ph/Baw2iOm0NJIsr3SA8B6
+VJ/JRRiaG4PW6TNJumiYVRZ7RCx+DnWveq+ombPDrjp7sbABx2s8tyIxFf7IW2wb
+bOHA2FNjRu2+gd6PNVlNYVX+i0tx+q8f7yuCOxxSux5DvfK5p+TE2XqWoc3TbtqZ
+9SXKeoUVDvhWjY7nH8D0LehYobh0LzCIpdsm1wqDpR3DOMlNAY+5OHA0+4BAoY+K
+hPOyWA5TWLrlC6yMuz6ttVFvjOHGd7HwLqlU3NU4aj9oxLIjaHHHOcxvf+v0GrvL
+QTd7YfyFeEpMQjwK/ueYlswxRqMHMFVLg/F0sAcwJ4W3F9yakz8mS341IDSWPoUR
+Kc1KArFH4SqW2oc3ChzFwU/Om7rv5rZelJRRTzW8lJZFPysiLtUoEe1PeiYiikCL
+Am40XX62EF+MG+TKQkYkTGlSPjNEE+ko9cQcAzMWVz1BZCUjcJH+dI/7uNsMOtp3
+y1ytX+Asv2KiLGAKzQvKBIrJzYuwuStVxUIZguJpb20bo8CkLOUKORjwsiQ4ktna
+gHoN2GLHvDzOq4cixubg
+=65Dw
+-----END PGP SIGNATURE-----
--- a/contrib/FastSync/load-utxo-set.sh
+++ b/contrib/FastSync/load-utxo-set.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# This script shows the steps to download and deploy an archive of the current UTXO Set
+# It will:
+#   1. Download the UTXO Set from UTXO_DOWNLOAD_LINK, if UTXO_DOWNLOAD_LINK is empty, use NBITCOIN_NETWORK to find a default
+#   2. Check the tarball against trusted hashes
+#   3. Create the container's folders for blocks and chainstate, or empty them if they exists
+#   4. Unzip the tarball
+
+if ! [ "$0" = "$BASH_SOURCE" ]; then
+    echo "This script must not be sourced" 
+    exit 1
+fi
+
+if [[ $EUID -ne 0 ]]; then
+   echo "This script must be run as root after running \"sudo su -\"" 
+   exit 1
+fi
+
+if ! [[ "$NBITCOIN_NETWORK" ]]; then
+    echo "NBITCOIN_NETWORK should be set to mainnet, testnet or regtest" 
+    exit 1
+fi
+
+if ! [[ "$UTXO_DOWNLOAD_LINK" ]]; then
+    [[ $NBITCOIN_NETWORK == "mainnet" ]] && UTXO_DOWNLOAD_LINK="http://utxosets.blob.core.windows.net/public/utxo-snapshot-bitcoin-mainnet-551636.tar"
+    [[ $NBITCOIN_NETWORK == "testnet" ]] && UTXO_DOWNLOAD_LINK="http://utxosets.blob.core.windows.net/public/utxo-snapshot-bitcoin-testnet-1445586.tar"
+fi
+
+if ! [[ "$UTXO_DOWNLOAD_LINK" ]]; then
+    echo "No default UTXO_DOWNLOAD_LINK for $NBITCOIN_NETWORK" 
+    exit 1
+fi
+
+BITCOIN_DATA_DIR="/var/lib/docker/volumes/generated_bitcoin_datadir/_data"
+[ ! -d "$BITCOIN_DATA_DIR" ] && mkdir -p "$BITCOIN_DATA_DIR"
+
+TAR_NAME="$(basename $UTXO_DOWNLOAD_LINK)"
+TAR_FILE="$BITCOIN_DATA_DIR/$TAR_NAME"
+
+cp "utxo-sets" "$BITCOIN_DATA_DIR/utxo-sets"
+cd "$BITCOIN_DATA_DIR"
+if [ ! -f "$TAR_FILE" ]; then
+  echo "Downloading $UTXO_DOWNLOAD_LINK to $TAR_FILE"
+  wget "$UTXO_DOWNLOAD_LINK" -q --show-progress
+else
+  echo "$TAR_FILE already exists"
+fi
+
+grep "$TAR_NAME" "utxo-sets" | tee "utxo-set"
+rm "utxo-sets"
+if ! sha256sum -c "utxo-set"; then
+  echo "$TAR_FILE is not trusted"
+  rm "utxo-set"
+  cd -
+  exit 1
+fi
+rm "utxo-set"
+cd -
+
+NETWORK_DIRECTORY=$NBITCOIN_NETWORK
+if [[ $NBITCOIN_NETWORK == "mainnet" ]]; then
+  NETWORK_DIRECTORY="."
+fi
+if [[ $NBITCOIN_NETWORK == "testnet" ]]; then
+  NETWORK_DIRECTORY="testnet3"
+fi
+
+NETWORK_DIRECTORY="$BITCOIN_DATA_DIR/$NETWORK_DIRECTORY"
+[ -d "$NETWORK_DIRECTORY/blocks" ] && rm -rf "$NETWORK_DIRECTORY/blocks"
+[ -d "$NETWORK_DIRECTORY/chainstate" ] && rm -rf "$NETWORK_DIRECTORY/chainstate"
+[ ! -d "$NETWORK_DIRECTORY" ] && mkdir "$NETWORK_DIRECTORY"
+
+echo "Extracting..."
+if ! tar -xf "$TAR_FILE" -C "$BITCOIN_DATA_DIR"; then
+  echo "Failed extracting, did you turned bitcoin off? (btcpay-down.sh)"
+  exit 1
+fi
+rm "$TAR_FILE"
+
+BTCPAY_DATA_DIR="/var/lib/docker/volumes/generated_btcpay_datadir/_data"
+[ ! -d "$BTCPAY_DATA_DIR" ] && mkdir -p "$BTCPAY_DATA_DIR"
+echo "$TAR_NAME" > "$BTCPAY_DATA_DIR/FastSynced"
+
+echo "Successfully downloaded and extracted, you can run btcpay again (btcpay-up.sh)"
--- a/contrib/FastSync/save-utxo-set-in-bitcoind.sh
+++ b/contrib/FastSync/save-utxo-set-in-bitcoind.sh
@@ -0,0 +1,30 @@
+# This file is internal and meant to be run by save-utxo-set.sh
+BITCOIND="bitcoind -datadir=/data"
+BITCOIN_CLI="bitcoin-cli -datadir=/data"
+
+$BITCOIND &
+BITCOIND_PID=$!
+CURRENT_HEIGHT="$($BITCOIN_CLI -rpcwait getblockcount)"
+let "PRUNED_HEIGHT=$CURRENT_HEIGHT - 289"
+
+echo "Pruning to $PRUNED_HEIGHT"
+$BITCOIN_CLI pruneblockchain "$PRUNED_HEIGHT"
+
+echo "Waiting bitcoind to stop..."
+$BITCOIN_CLI stop
+wait $BITCOIND_PID
+
+NETWORK_DIRECTORY=$NBITCOIN_NETWORK
+if [[ $NBITCOIN_NETWORK == "mainnet" ]]; then
+  NETWORK_DIRECTORY="."
+fi
+if [[ $NBITCOIN_NETWORK == "testnet" ]]; then
+  NETWORK_DIRECTORY="testnet3"
+fi
+
+cd /data
+TAR_NAME="utxo-snapshot-bitcoin-$NBITCOIN_NETWORK-$PRUNED_HEIGHT.tar"
+echo "Creating $TAR_NAME..."
+tar -cf "$TAR_NAME" "$NETWORK_DIRECTORY/blocks/"
+tar -rf "$TAR_NAME" "$NETWORK_DIRECTORY/chainstate/"
+exit
--- a/contrib/FastSync/save-utxo-set.sh
+++ b/contrib/FastSync/save-utxo-set.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# This script shows the steps to create an archive of the current UTXO Set
+# It will:
+#   1. Shutdown BTCPay Server
+#   2. Start bitcoind
+#   3. Prune it to up to 289 blocks from the tip
+#   4. Stop bitcoind
+#   5. Archive in a tarball the blocks and chainstate directories
+#   6. Restart BTCPay
+#   7. If AZURE_STORAGE_CONNECTION_STRING is set, then upload to azure storage and make the blob public, else print hash and tarball
+
+: "${AZURE_STORAGE_CONTAINER:=public}"
+
+btcpay-down.sh
+
+for i in /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-*; do
+    echo "Deleting $i"
+    rm $i
+done
+
+rm /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-*
+# Run only bitcoind and connect to it
+SCRIPT="$(cat save-utxo-set-in-bitcoind.sh)"
+cd "`dirname $BTCPAY_ENV_FILE`"
+docker-compose -f $BTCPAY_DOCKER_COMPOSE run -e "NBITCOIN_NETWORK=$NBITCOIN_NETWORK" bitcoind bash -c "$SCRIPT"
+btcpay-up.sh
+
+echo "Calculating the hash of the tar file..."
+TAR_FILE="$(echo /var/lib/docker/volumes/generated_bitcoin_datadir/_data/utxo-snapshot-*)"
+TAR_FILE_HASH="$(sha256sum "$TAR_FILE" | cut -d " " -f 1)"
+
+if [[ "$AZURE_STORAGE_CONNECTION_STRING" ]]; then
+    echo "Uploading to azure..."
+    # Install az from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest
+
+    BLOB_NAME="$(basename -- $TAR_FILE)"
+    az storage container create --name "$AZURE_STORAGE_CONTAINER" --public-access "blob"
+    az storage blob upload -f "$TAR_FILE" \
+                        -c "$AZURE_STORAGE_CONTAINER" \
+                        -n "$BLOB_NAME" \
+                        --content-type "application/x-tar"
+
+    az storage blob metadata update --container-name "$AZURE_STORAGE_CONTAINER" --name "$BLOB_NAME" --metadata "sha256=$TAR_FILE_HASH"
+
+    # Print the sha256sum. Downloaders will need to verify this
+    STORAGE_URL="$(az storage blob url --container-name "$AZURE_STORAGE_CONTAINER" --name "$BLOB_NAME" --protocol "http")"
+    echo "You can now download the UTXO on $STORAGE_URL"
+    echo "Please, after download, verify the sha256 with:"
+    echo "echo "$TAR_FILE_HASH  $BLOB_NAME" | sha256sum -c -"
+    rm "$TAR_FILE"
+else
+    echo "SHA256: $TAR_FILE_HASH"
+    echo "File at: $TAR_FILE"
+fi
--- a/contrib/FastSync/utxo-sets
+++ b/contrib/FastSync/utxo-sets
@@ -0,0 +1,2 @@
+fab994299273080bf7124c8c45c4ada867974ca747900178496a69e450cf713f  utxo-snapshot-bitcoin-mainnet-551636.tar
+eabaaa717bb8eeaf603e383dd8642d9d34df8e767fccbd208b0c936b79c82742  utxo-snapshot-bitcoin-testnet-1445586.tar