Automated BIP39 Recovery, squashed

electrum/base_wizard.py

@@ -34,7 +34,7 @@ from . import bitcoin
 from . import keystore
 from . import mnemonic
 from .bip32 import is_bip32_derivation, xpub_type, normalize_bip32_derivation, BIP32Node
-from .keystore import bip44_derivation, purpose48_derivation, Hardware_KeyStore, KeyStore
+from .keystore import bip44_derivation, purpose48_derivation, Hardware_KeyStore, KeyStore, bip39_to_seed
 from .wallet import (Imported_Wallet, Standard_Wallet, Multisig_Wallet,
                      wallet_types, Wallet, Abstract_Wallet)
 from .storage import WalletStorage, StorageEncryptionVersion
@@ -404,7 +404,7 @@ class BaseWizard(Logger):
         else:
             raise Exception('unknown purpose: %s' % purpose)
 
-    def derivation_and_script_type_dialog(self, f):
+    def derivation_and_script_type_dialog(self, f, get_account_xpub=None):
         message1 = _('Choose the type of addresses in your wallet.')
         message2 = ' '.join([
             _('You can override the suggested derivation path.'),
@@ -429,10 +429,10 @@ class BaseWizard(Logger):
             ]
         while True:
             try:
-                self.choice_and_line_dialog(
+                self.derivation_and_script_type_gui_specific_dialog(
                     run_next=f, title=_('Script type and Derivation path'), message1=message1,
                     message2=message2, choices=choices, test_text=is_bip32_derivation,
-                    default_choice_idx=default_choice_idx)
+                    default_choice_idx=default_choice_idx, get_account_xpub=get_account_xpub)
                 return
             except ScriptTypeNotSupported as e:
                 self.show_error(e)
@@ -492,7 +492,8 @@ class BaseWizard(Logger):
     def on_restore_seed(self, seed, is_bip39, is_ext):
         self.seed_type = 'bip39' if is_bip39 else mnemonic.seed_type(seed)
         if self.seed_type == 'bip39':
-            f = lambda passphrase: self.on_restore_bip39(seed, passphrase)
+            def f(passphrase):
+                self.on_restore_bip39(seed, passphrase)
             self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('')
         elif self.seed_type in ['standard', 'segwit']:
             f = lambda passphrase: self.run('create_keystore', seed, passphrase)
@@ -509,7 +510,13 @@ class BaseWizard(Logger):
         def f(derivation, script_type):
             derivation = normalize_bip32_derivation(derivation)
             self.run('on_bip43', seed, passphrase, derivation, script_type)
-        self.derivation_and_script_type_dialog(f)
+        def get_account_xpub(account_path):
+            root_seed = bip39_to_seed(seed, passphrase)
+            root_node = BIP32Node.from_rootseed(root_seed, xtype="standard")
+            account_node = root_node.subkey_at_private_derivation(account_path)
+            account_xpub = account_node.to_xpub()
+            return account_xpub
+        self.derivation_and_script_type_dialog(f, get_account_xpub)
 
     def create_keystore(self, seed, passphrase):
         k = keystore.from_seed(seed, passphrase, self.wallet_type == 'multisig')

electrum/bip39_recovery.py

@@ -0,0 +1,66 @@
+# Copyright (C) 2020 The Electrum developers
+# Distributed under the MIT software license, see the accompanying
+# file LICENCE or http://www.opensource.org/licenses/mit-license.php
+
+from aiorpcx import TaskGroup
+
+from . import bitcoin
+from .constants import BIP39_WALLET_FORMATS
+from .bip32 import BIP32_PRIME, BIP32Node
+from .bip32 import convert_bip32_path_to_list_of_uint32 as bip32_str_to_ints
+from .bip32 import convert_bip32_intpath_to_strpath as bip32_ints_to_str
+
+async def account_discovery(network, get_account_xpub):
+    async with TaskGroup() as group:
+        account_scan_tasks = []
+        for wallet_format in BIP39_WALLET_FORMATS:
+            account_scan = scan_for_active_accounts(network, get_account_xpub, wallet_format)
+            account_scan_tasks.append(await group.spawn(account_scan))
+    active_accounts = []
+    for task in account_scan_tasks:
+        active_accounts.extend(task.result())
+    return active_accounts
+
+async def scan_for_active_accounts(network, get_account_xpub, wallet_format):
+    active_accounts = []
+    account_path = bip32_str_to_ints(wallet_format["derivation_path"])
+    while True:
+        account_xpub = get_account_xpub(account_path)
+        account_node = BIP32Node.from_xkey(account_xpub)
+        has_history = await account_has_history(network, account_node, wallet_format["script_type"]);
+        if has_history:
+            account = format_account(wallet_format, account_path)
+            active_accounts.append(account)
+        if not has_history or not wallet_format["iterate_accounts"]:
+            break
+        account_path[-1] = account_path[-1] + 1
+    return active_accounts
+
+async def account_has_history(network, account_node, script_type):
+    gap_limit = 20
+    async with TaskGroup() as group:
+        get_history_tasks = []
+        for address_index in range(gap_limit):
+            address_node = account_node.subkey_at_public_derivation("0/" + str(address_index))
+            pubkey = address_node.eckey.get_public_key_hex()
+            address = bitcoin.pubkey_to_address(script_type, pubkey)
+            script = bitcoin.address_to_script(address)
+            scripthash = bitcoin.script_to_scripthash(script)
+            get_history = network.get_history_for_scripthash(scripthash)
+            get_history_tasks.append(await group.spawn(get_history))
+    for task in get_history_tasks:
+        history = task.result()
+        if len(history) > 0:
+            return True
+    return False
+
+def format_account(wallet_format, account_path):
+    description = wallet_format["description"]
+    if wallet_format["iterate_accounts"]:
+        account_index = account_path[-1] % BIP32_PRIME
+        description = f'{description} (Account {account_index})'
+    return {
+        "description": description,
+        "derivation_path": bip32_ints_to_str(account_path),
+        "script_type": wallet_format["script_type"],
+    }

electrum/bip39_wallet_formats.json

@@ -0,0 +1,80 @@
+[
+    {
+        "description": "Standard BIP44 legacy",
+        "derivation_path": "m/44'/0'/0'",
+        "script_type": "p2pkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Standard BIP49 compatibility segwit",
+        "derivation_path": "m/49'/0'/0'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Standard BIP84 native segwit",
+        "derivation_path": "m/84'/0'/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard legacy",
+        "derivation_path": "m/0'",
+        "script_type": "p2pkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard compatibility segwit",
+        "derivation_path": "m/0'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard native segwit",
+        "derivation_path": "m/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Copay native segwit",
+        "derivation_path": "m/44'/0'/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Samourai Bad Bank (toxic change)",
+        "derivation_path": "m/84'/0'/2147483644'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Whirlpool Pre Mix",
+        "derivation_path": "m/84'/0'/2147483645'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Whirlpool Post Mix",
+        "derivation_path": "m/84'/0'/2147483646'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet legacy",
+        "derivation_path": "m/44'/0'/2147483647'",
+        "script_type": "p2pkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet compatibility segwit",
+        "derivation_path": "m/49'/0'/2147483647'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet native segwit",
+        "derivation_path": "m/84'/0'/2147483647'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    }
+]

electrum/constants.py

@@ -42,6 +42,7 @@ def read_json(filename, default):
 
 GIT_REPO_URL = "https://github.com/spesmilo/electrum"
 GIT_REPO_ISSUES_URL = "https://github.com/spesmilo/electrum/issues"
+BIP39_WALLET_FORMATS = read_json('bip39_wallet_formats.json', [])
 
 
 class AbstractNet:

electrum/gui/kivy/uix/dialogs/installwizard.py

@@ -1115,7 +1115,7 @@ class InstallWizard(BaseWizard, Widget):
     def multisig_dialog(self, **kwargs): WizardMultisigDialog(self, **kwargs).open()
     def show_seed_dialog(self, **kwargs): ShowSeedDialog(self, **kwargs).open()
     def line_dialog(self, **kwargs): LineDialog(self, **kwargs).open()
-    def choice_and_line_dialog(self, **kwargs): ChoiceLineDialog(self, **kwargs).open()
+    def derivation_and_script_type_gui_specific_dialog(self, **kwargs): ChoiceLineDialog(self, **kwargs).open()
 
     def confirm_seed_dialog(self, **kwargs):
         kwargs['title'] = _('Confirm Seed')

electrum/gui/qt/bip39_recovery_dialog.py

@@ -0,0 +1,64 @@
+# Copyright (C) 2020 The Electrum developers
+# Distributed under the MIT software license, see the accompanying
+# file LICENCE or http://www.opensource.org/licenses/mit-license.php
+
+from PyQt5.QtCore import Qt
+from PyQt5.QtWidgets import QWidget, QVBoxLayout, QGridLayout, QLabel, QListWidget, QListWidgetItem
+
+from electrum.i18n import _
+from electrum.network import Network
+from electrum.bip39_recovery import account_discovery
+
+from .util import WindowModalDialog, MessageBoxMixin, TaskThread, Buttons, CancelButton, OkButton
+
+class Bip39RecoveryDialog(WindowModalDialog):
+    def __init__(self, parent: QWidget, get_account_xpub, on_account_select):
+        self.get_account_xpub = get_account_xpub
+        self.on_account_select = on_account_select
+        WindowModalDialog.__init__(self, parent, _('BIP39 Recovery'))
+        self.setMinimumWidth(400)
+        vbox = QVBoxLayout(self)
+        self.content = QVBoxLayout()
+        self.content.addWidget(QLabel(_('Scanning common paths for existing accounts...')))
+        vbox.addLayout(self.content)
+        self.ok_button = OkButton(self)
+        self.ok_button.clicked.connect(self.on_ok_button_click)
+        self.ok_button.setEnabled(False)
+        vbox.addLayout(Buttons(CancelButton(self), self.ok_button))
+        self.show()
+        self.thread = TaskThread(self)
+        self.thread.finished.connect(self.deleteLater) # see #3956
+        self.thread.add(self.recovery, self.on_recovery_success, None, self.on_recovery_error)
+
+    def on_ok_button_click(self):
+        item = self.list.currentItem()
+        account = item.data(Qt.UserRole)
+        self.on_account_select(account)
+
+    def recovery(self):
+        network = Network.get_instance()
+        coroutine = account_discovery(network, self.get_account_xpub)
+        return network.run_from_another_thread(coroutine)
+
+    def on_recovery_success(self, accounts):
+        self.clear_content()
+        if len(accounts) == 0:
+            self.content.addWidget(QLabel(_('No existing accounts found.')))
+            return
+        self.content.addWidget(QLabel(_('Choose an account to restore.')))
+        self.list = QListWidget()
+        for account in accounts:
+            item = QListWidgetItem(account['description'])
+            item.setData(Qt.UserRole, account)
+            self.list.addItem(item)
+        self.list.clicked.connect(lambda: self.ok_button.setEnabled(True))
+        self.content.addWidget(self.list)
+
+    def on_recovery_error(self, error):
+        self.clear_content()
+        self.content.addWidget(QLabel(_('Error: Account discovery failed.')))
+        print(error)
+
+    def clear_content(self):
+        for i in reversed(range(self.content.count())):
+            self.content.itemAt(i).widget().setParent(None)

electrum/gui/qt/installwizard.py

@@ -28,6 +28,7 @@ from .network_dialog import NetworkChoiceLayout
 from .util import (MessageBoxMixin, Buttons, icon_path, ChoicesLayout, WWLabel,
                    InfoButton, char_width_in_lineedit, PasswordLineEdit)
 from .password_dialog import PasswordLayout, PasswordLayoutForHW, PW_NEW
+from .bip39_recovery_dialog import Bip39RecoveryDialog
 from electrum.plugin import run_hook, Plugins
 
 if TYPE_CHECKING:
@@ -603,11 +604,25 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         return clayout.selected_index()
 
     @wizard_dialog
-    def choice_and_line_dialog(self, title: str, message1: str, choices: List[Tuple[str, str, str]],
+    def derivation_and_script_type_gui_specific_dialog(self, title: str, message1: str, choices: List[Tuple[str, str, str]],
                                message2: str, test_text: Callable[[str], int],
-                               run_next, default_choice_idx: int=0) -> Tuple[str, str]:
+                               run_next, default_choice_idx: int=0, get_account_xpub=None) -> Tuple[str, str]:
         vbox = QVBoxLayout()
 
+        if get_account_xpub:
+            button = QPushButton(_("Detect Existing Accounts"))
+            def on_account_select(account):
+                script_type = account["script_type"]
+                if script_type == "p2pkh":
+                    script_type = "standard"
+                button_index = c_values.index(script_type)
+                button = clayout.group.buttons()[button_index]
+                button.setChecked(True)
+                line.setText(account["derivation_path"])
+            button.clicked.connect(lambda: Bip39RecoveryDialog(self, get_account_xpub, on_account_select))
+            vbox.addWidget(button, alignment=Qt.AlignLeft)
+            vbox.addWidget(QLabel(_("Or")))
+
         c_values = [x[0] for x in choices]
         c_titles = [x[1] for x in choices]
         c_default_text = [x[2] for x in choices]
@@ -618,7 +633,6 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
                                 checked_index=default_choice_idx)
         vbox.addLayout(clayout.layout())
 
-        vbox.addSpacing(50)
         vbox.addWidget(WWLabel(message2))
 
         line = QLineEdit()

electrum/scripts/bip39_recovery.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import sys
+import asyncio
+
+from electrum.util import json_encode, print_msg, create_and_start_event_loop, log_exceptions
+from electrum.simple_config import SimpleConfig
+from electrum.network import Network
+from electrum.keystore import bip39_to_seed
+from electrum.bip32 import BIP32Node
+from electrum.bip39_recovery import account_discovery
+
+try:
+    mnemonic = sys.argv[1]
+    passphrase = sys.argv[2] if len(sys.argv) > 2 else ""
+except Exception:
+    print("usage: bip39_recovery <mnemonic> [<passphrase>]")
+    sys.exit(1)
+
+loop, stopping_fut, loop_thread = create_and_start_event_loop()
+
+config = SimpleConfig()
+network = Network(config)
+network.start()
+
+@log_exceptions
+async def f():
+    try:
+        def get_account_xpub(account_path):
+            root_seed = bip39_to_seed(mnemonic, passphrase)
+            root_node = BIP32Node.from_rootseed(root_seed, xtype="standard")
+            account_node = root_node.subkey_at_private_derivation(account_path)
+            account_xpub = account_node.to_xpub()
+            return account_xpub
+        active_accounts = await account_discovery(network, get_account_xpub)
+        print_msg(json_encode(active_accounts))
+    finally:
+        stopping_fut.set_result(1)
+
+asyncio.run_coroutine_threadsafe(f(), loop)