ln: add test_lnwatcher

2019-01-23 15:10:11 +01:00
parent 129099797a
commit 8fc1779b0d
10 changed files with 237 additions and 10 deletions
--- a/electrum/lnbase.py
+++ b/electrum/lnbase.py
@@ -16,6 +16,7 @@ import sys

 import aiorpcx

+from .simple_config import get_config
 from .crypto import sha256, sha256d
 from . import bitcoin
 from . import ecc
@@ -723,9 +724,12 @@ class Peer(PrintError):
        if chan.config[LOCAL].funding_locked_received:
            self.mark_open(chan)
        else:
-            self.print_error("remote hasn't sent funding_locked, disconnecting (should reconnect again shortly)")
-            self.close_and_cleanup()
-            self.network.trigger_callback('channel', chan)
+            # only when not testing since this is an issue
+            # only with lnd and the test uses electrum
+            if not get_config().debug_lightning:
+                self.print_error("remote hasn't sent funding_locked, disconnecting (should reconnect again shortly)")
+                self.close_and_cleanup()
+                self.network.trigger_callback('channel', chan)

    def on_funding_locked(self, payload):
        channel_id = payload['channel_id']
@@ -1087,7 +1091,9 @@ class Peer(PrintError):
            return
        self.network.trigger_callback('htlc_added', UpdateAddHtlc(**htlc, htlc_id=htlc_id), invoice, RECEIVED)
        # settle htlc
-        await self.settle_htlc(chan, htlc_id, preimage)
+        if not self.network.config.debug_lightning_do_not_settle:
+            # settle htlc
+            await self.settle_htlc(chan, htlc_id, preimage)

    async def settle_htlc(self, chan: Channel, htlc_id: int, preimage: bytes):
        chan.settle_htlc(preimage, htlc_id)
--- a/electrum/lnchan.py
+++ b/electrum/lnchan.py
@@ -22,6 +22,7 @@
 # API (method signatures and docstrings) partially copied from lnd
 # 42de4400bff5105352d0552155f73589166d162b

+import os
 from collections import namedtuple, defaultdict
 import binascii
 import json
@@ -176,6 +177,11 @@ class Channel(PrintError):
        self.local_commitment = ctx
        if self.sweep_address is not None:
            self.local_sweeptxs = create_sweeptxs_for_our_latest_ctx(self, self.local_commitment, self.sweep_address)
+            initial = os.path.join(get_config().electrum_path(), 'initial_commitment_tx')
+            tx = self.force_close_tx().serialize_to_network()
+            if not os.path.exists(initial):
+                with open(initial, 'w') as f:
+                    f.write(tx)

    def set_remote_commitment(self):
        self.remote_commitment = self.current_commitment(REMOTE)
--- a/electrum/lnsweep.py
+++ b/electrum/lnsweep.py
@@ -95,7 +95,7 @@ def create_sweeptxs_for_their_just_revoked_ctx(chan: 'Channel', ctx: Transaction
            is_revocation=True)
        # sweep from htlc tx
        secondstage_sweep_tx = create_sweeptx_that_spends_htlctx_that_spends_htlc_in_ctx(
-            'their_htlctx_',
+            'sweep_from_their_ctx_htlc_',
            to_self_delay=0,
            htlc_tx=htlc_tx,
            htlctx_witness_script=htlc_tx_witness_script,
@@ -172,7 +172,7 @@ def create_sweeptxs_for_our_latest_ctx(chan: 'Channel', ctx: Transaction,
            preimage=preimage,
            is_received_htlc=is_received_htlc)
        to_wallet_tx = create_sweeptx_that_spends_htlctx_that_spends_htlc_in_ctx(
-            'our_ctx_htlc_tx_',
+            'sweep_from_our_ctx_htlc_',
            to_self_delay=to_self_delay,
            htlc_tx=htlc_tx,
            htlctx_witness_script=htlctx_witness_script,
--- a/electrum/lnworker.py
+++ b/electrum/lnworker.py
@@ -72,9 +72,9 @@ class LNWorker(PrintError):
        self.network = network
        self.channel_db = self.network.channel_db
        self.lock = threading.RLock()
+        self.config = network.config
        self.ln_keystore = self._read_ln_keystore()
        self.node_keypair = generate_keypair(self.ln_keystore, LnKeyFamily.NODE_KEY, 0)
-        self.config = network.config
        self.peers = {}  # type: Dict[bytes, Peer]  # pubkey -> Peer
        self.invoices = wallet.storage.get('lightning_invoices', {})  # type: Dict[str, Tuple[str,str]]  # RHASH -> (preimage, invoice)
        self.channels = {}  # type: Dict[bytes, Channel]
@@ -209,9 +209,13 @@ class LNWorker(PrintError):
    def _read_ln_keystore(self) -> BIP32_KeyStore:
        xprv = self.wallet.storage.get('lightning_privkey2')
        if xprv is None:
-            # TODO derive this deterministically from wallet.keystore at keystore generation time
-            # probably along a hardened path ( lnd-equivalent would be m/1017'/coinType'/ )
-            seed = os.urandom(32)
+            if not self.config.debug_lightning:
+                # TODO derive this deterministically from wallet.keystore at keystore generation time
+                # probably along a hardened path ( lnd-equivalent would be m/1017'/coinType'/ )
+                seed = os.urandom(32)
+            else:
+                # dangerous deterministic secret for testing
+                seed = sha256(self.config.electrum_path())
            xprv, xpub = bip32_root(seed, xtype='standard')
            self.wallet.storage.put('lightning_privkey2', xprv)
            self.wallet.storage.write()
--- a/electrum/simple_config.py
+++ b/electrum/simple_config.py
@@ -67,6 +67,9 @@ class SimpleConfig(Logger):

        Logger.__init__(self)

+        self.debug_lightning = 'ELECTRUM_DEBUG_LIGHTNING_DANGEROUS' in os.environ
+        self.debug_lightning_do_not_settle = 'ELECTRUM_DEBUG_LIGHTNING_DO_NOT_SETTLE' in os.environ
+
        # This lock needs to be acquired for updating and reading the config in
        # a thread-safe way.
        self.lock = threading.RLock()
--- a/electrum/tests/test_lnwatcher.py
+++ b/electrum/tests/test_lnwatcher.py
@@ -0,0 +1,42 @@
+import base64
+import os
+import sys
+import unittest
+import subprocess
+from electrum.crypto import sha256
+from electrum.util import bh2u
+import itertools
+import pathlib
+
+def split_seq(iterable, size):
+    it = iter(iterable)
+    item = list(itertools.islice(it, size))
+    while item:
+        yield item
+        item = list(itertools.islice(it, size))
+
+not_travis_text = 'breach test takes a long time, installs things, requires certain ports to be available, assumes x86 and doesn\'t clean up after itself'
+
+@unittest.skipUnless(os.getlogin() == 'travis', not_travis_text)
+class TestLNWatcher(unittest.TestCase):
+    maxDiff = None # unlimited
+
+    @staticmethod
+    def run_shell(cmd, timeout=60):
+        process = subprocess.Popen(['electrum/tests/test_lnwatcher/' + cmd[0]] + ([] if len(cmd) == 1 else cmd[1:]), stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+        for line in iter(process.stdout.readline, b''):
+            sys.stdout.write(line.decode(sys.stdout.encoding))
+        process.wait(timeout=timeout)
+        assert process.returncode == 0
+
+    @classmethod
+    def setUpClass(cls):
+        cls.run_shell(['setup.sh'])
+
+    def test_redeem_stuck_htlcs(self):
+        self.run_shell(['start_dependencies.sh', 'do_not_settle_elec2'])
+        self.run_shell(['redeem_htlcs.sh'])
+
+    def test_funder_publishes_initial_commitment_and_fundee_takes_all(self):
+        self.run_shell(['start_dependencies.sh'])
+        self.run_shell(['breach.sh'])
--- a/electrum/tests/test_lnwatcher/breach.sh
+++ b/electrum/tests/test_lnwatcher/breach.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+export HOME=~
+set -eux pipefail
+bitcoin-cli generatetoaddress 109 mwLZSJ2hUkvFoSkyadNGgmu9977w6K8wfj > /dev/null
+sleep 30
+othernode=$(./run_electrum --regtest -D /tmp/elec2 nodeid)
+./run_electrum --regtest -D /tmp/elec1 open_channel $othernode@localhost 0.15
+sleep 3
+bitcoin-cli generatetoaddress 6 mwLZSJ2hUkvFoSkyadNGgmu9977w6K8wfj > /dev/null
+sleep 12
+invoice=$(./run_electrum --regtest -D /tmp/elec2 addinvoice 0.01 invoice_description)
+timeout 5 ./run_electrum -D /tmp/elec1 --regtest lnpay $invoice || (cat screenlog*; exit 1)
+bitcoin-cli sendrawtransaction $(cat /tmp/elec1/regtest/initial_commitment_tx)
+# elec2 should take all funds because breach
+sleep 12
+bitcoin-cli generatetoaddress 2 mwLZSJ2hUkvFoSkyadNGgmu9977w6K8wfj > /dev/null
+sleep 12
+balance=$(./run_electrum --regtest -D /tmp/elec2 getbalance | jq '.confirmed | tonumber')
+if (( $(echo "$balance < 0.14" | bc -l) )); then
+    echo "balance of elec2 insufficient: $balance"
+    exit 1
+fi
--- a/electrum/tests/test_lnwatcher/redeem_htlcs.sh
+++ b/electrum/tests/test_lnwatcher/redeem_htlcs.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+export HOME=~
+set -eux pipefail
+bitcoin-cli generatetoaddress 100 bcrt1qxcjufgh2jarkp2qkx68azh08w9v5gah8u6es8s > /dev/null
+sleep 30
+balance_before=$(./run_electrum --regtest -D /tmp/elec1 getbalance | jq -r .confirmed)
+othernode=$(./run_electrum --regtest -D /tmp/elec2 nodeid)
+./run_electrum --regtest -D /tmp/elec1 open_channel $othernode@localhost 0.15
+sleep 12
+bitcoin-cli generatetoaddress 6 bcrt1qxcjufgh2jarkp2qkx68azh08w9v5gah8u6es8s > /dev/null
+sleep 12
+balance_during=$(./run_electrum --regtest -D /tmp/elec1 getbalance | jq -r .confirmed)
+if [[ "$balance_during" == "$balance_before" ]]; then
+    echo 'balance has not changed'
+    ./run_electrum --regtest -D /tmp/elec1 getbalance
+    exit 1
+fi
+for i in $(seq 0 0); do
+    invoice=$(./run_electrum --regtest -D /tmp/elec2 addinvoice 0.01 invoice_description$i)
+    ./run_electrum -D /tmp/elec1 --regtest lnpay $invoice
+done
+screen -S elec2 -X quit
+sleep 1
+ps ax | grep run_electrum
+chan_id=$(python3 run_electrum -D /tmp/elec1 --regtest listchannels | jq -r ".[0].channel_point" | cut -d: -f1)
+./run_electrum -D /tmp/elec1 --regtest closechannel $chan_id --force
+sleep 12
+bitcoin-cli generatetoaddress 144 bcrt1qxcjufgh2jarkp2qkx68azh08w9v5gah8u6es8s
+sleep 30
+bitcoin-cli generatetoaddress 10 bcrt1qxcjufgh2jarkp2qkx68azh08w9v5gah8u6es8s
+sleep 12
+bitcoin-cli generatetoaddress 10 bcrt1qxcjufgh2jarkp2qkx68azh08w9v5gah8u6es8s
+sleep 12
+balance_after_elec2=$(./run_electrum --regtest -D /tmp/elec2 getbalance |  jq '[.confirmed, .unconfirmed] | to_entries | map(select(.value != null).value) | map(tonumber) | add ')
+if [[ "$balance_after_elec2" != "0" ]]; then
+    echo 'elec2 has balance, DO_NOT_SETTLE did not work'
+    exit 1
+fi
+date
+./run_electrum --regtest -D /tmp/elec1 history --show_fees
+balance_after=$(./run_electrum --regtest -D /tmp/elec1 getbalance |  jq '[.confirmed, .unconfirmed] | to_entries | map(select(.value != null).value) | map(tonumber) | add ')
+if (( $(echo "$balance_after - $balance_during < 0.14" | bc -l) || $(echo "$balance_after - $balance_during > 0.15" | bc -l) )); then
+    echo "balance of elec1 not between 0.14 and 0.15, to_local and htlcs not redeemed. balance was $balance_before before, $balance_during after channel opening and $balance_after after force closing"
+    tail -n 200 screenlog.0
+    exit 1
+fi
--- a/electrum/tests/test_lnwatcher/setup.sh
+++ b/electrum/tests/test_lnwatcher/setup.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -eux pipefail
+#this is downloading so much, and all we need is bitcoin-cli
+#if [ ! -f bitcoin-*.tar.gz ]; then
+#    wget https://bitcoin.org/bin/bitcoin-core-0.17.0.1/bitcoin-0.17.0.1-x86_64-linux-gnu.tar.gz
+#fi
+#tar xf bitcoin-*.tar.gz
+#sudo mv bitcoin-0.17.0/bin/bitcoin-cli /usr/bin/
+sudo wget -qO /usr/bin/bitcoin-cli https://sr.ht/e6-xS.bitcoincli # this is just bitcoin-cli from the 0.17.0.1 release, rehosted
+sudo chmod +x /usr/bin/bitcoin-cli
+sudo apt-get -qq update
+sudo apt-get -qq install libssl1.0.0 jq netcat lsof moreutils
+
+sudo curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
+sudo python3 get-pip.py
+python3 -m pip install plyvel pylru
+mkdir ~/.bitcoin
+cat > ~/.bitcoin/bitcoin.conf <<EOF
+regtest=1
+txindex=1
+printtoconsole=1
+rpcuser=doggman
+rpcpassword=donkey
+rpcallowip=127.0.0.1
+zmqpubrawblock=tcp://127.0.0.1:28332
+zmqpubrawtx=tcp://127.0.0.1:28333
+[regtest]
+rpcbind=0.0.0.0
+rpcport=18554
+EOF
+echo setup.sh done
--- a/electrum/tests/test_lnwatcher/start_dependencies.sh
+++ b/electrum/tests/test_lnwatcher/start_dependencies.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+export HOME=~
+ARG=$1
+set -eux pipefail
+if [ ! -f deterministic-bitcoind-ef70f9b5 ]; then
+    wget -q https://github.com/ysangkok/electrum-lightning-test/releases/download/v1/deterministic-bitcoind-ef70f9b5
+    chmod +x deterministic-bitcoind-ef70f9b5
+fi
+if [ ! -d electrumx ]; then
+    rm -f master.zip
+    wget -q https://github.com/kyuupichan/electrumx/archive/master.zip
+    unzip -q master.zip
+    mv electrumx-master electrumx
+fi
+screen -S bitcoind -X quit || true
+killall -9 deterministic-bitcoind-ef70f9b5 || true
+sleep 1
+ls -ld /.bitcoin/regtest || true
+rm -rf /.bitcoin/regtest
+ps -ef | grep bitcoin # bitcoin can rename itself to bitcoin-shutoff
+screen -S bitcoind -m -d ./deterministic-bitcoind-ef70f9b5 -regtest
+block_hash=""
+while [[ "$block_hash" == "" ]]; do
+    sleep 1
+    block_hash=$(bitcoin-cli generatetoaddress 1 mwLZSJ2hUkvFoSkyadNGgmu9977w6K8wfj | jq -r ".[0]" || true)
+done
+if [[ "$ARG" != "no_determinism" ]]; then
+    if [[ "$block_hash" != "40fc46e8bd87c0448ceb490b5339be674b89364c9f557e17b74b437d85b0a99c" ]]; then
+        echo 'not using deterministic bitcoind'
+        exit 1
+    fi
+fi
+screen -S electrumx -X quit || true
+kill -9 $(lsof -i :51001 -Fp | grep ^p | cut -c 2-) || true
+sleep 1
+screen -S electrumx -m -d bash -c "cd electrumx && rm -rf electrumx-db; mkdir electrumx-db && COIN=BitcoinSegwit TCP_PORT=51001 RPC_PORT=8000 NET=regtest DAEMON_URL=http://doggman:donkey@127.0.0.1:18554 DB_DIRECTORY=\$PWD/electrumx-db ./electrumx_server"
+sleep 5
+block_header_1="0000002006226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f68514a449a154326a7eafa4a6c6fc09639afe2dedde351446a790ec72af01b74dbe5494dffff7f2000000000"
+if [[ "$ARG" != "no_determinism" ]]; then
+    electrumx_header_1=""
+    while [[ "$electrumx_header_1" != "$block_header_1" ]]; do
+        sleep 3
+        electrumx_header_1=$(printf '{"id": 1, "method": "server.version", "params": ["testing", "1.4"]}\n{"id": 2, "method": "blockchain.block.header", "params": [1]}\n' | nc localhost 51001 | grep -v ElectrumX | jq -r .result)
+    done
+fi
+screen -S elec1 -X quit || true
+screen -S elec2 -X quit || true
+kill -9 $(ps ax | grep run_electrum | grep -v grep | awk '{print $1}') || true
+#kill -9 $(lsof -i :$(cat /tmp/elec1/regtest/daemon | python3 -c 'import ast, sys; print(ast.literal_eval(sys.stdin.read())[0][1])') -Fp | grep ^p | cut -c 2-) || true
+#kill -9 $(lsof -i :$(cat /tmp/elec2/regtest/daemon | python3 -c 'import ast, sys; print(ast.literal_eval(sys.stdin.read())[0][1])') -Fp | grep ^p | cut -c 2-) || true
+sleep 1
+rm -rf /tmp/elec?
+./run_electrum --regtest -D /tmp/elec1 restore "escape pumpkin perfect question nice all trigger course dismiss pole swallow burden"
+./run_electrum --regtest -D /tmp/elec2 restore "sure razor enrich panda sustain shoe napkin brick song van embark wave"
+cat > /tmp/elec2/regtest/config <<EOF
+{"lightning_listen": "127.0.0.1:9735"}
+EOF
+ELECTRUM_DEBUG_LIGHTNING_DANGEROUS=1 screen -L -d -m -S elec1 sh -c './run_electrum --regtest -D /tmp/elec1 daemon -v -s localhost:51001:t 2>&1 | ts'
+if [[ "$ARG" == "do_not_settle_elec2" ]]; then
+    ELECTRUM_DEBUG_LIGHTNING_DO_NOT_SETTLE=1 ELECTRUM_DEBUG_LIGHTNING_DANGEROUS=1 screen -L -d -m -S elec2 sh -c './run_electrum --regtest -D /tmp/elec2 daemon -v -s localhost:51001:t 2>&1 | ts'
+else
+    ELECTRUM_DEBUG_LIGHTNING_DANGEROUS=1 screen -L -d -m -S elec2 sh -c './run_electrum --regtest -D /tmp/elec2 daemon -v -s localhost:51001:t 2>&1 | ts'
+fi
+sleep 3
+./run_electrum --regtest -D /tmp/elec1 daemon load_wallet
+./run_electrum --regtest -D /tmp/elec2 daemon load_wallet
+sleep 3