AaronFiore/electrum
1
0
Fork 0
Code Activity

persist nodes in channel_db on disk

Browse Source
This commit is contained in:
SomberNight
2018-07-26 21:08:25 +02:00
committed by ThomasV
parent 5a05a92b3d
commit bc06ded4b9
8 changed files with 204 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
electrum/gui/qt/channels_list.py
View File

@@ -64,10 +64,12 @@ class ChannelsList(MyTreeWidget):
return h
def update_status(self):
n = len(self.parent.network.lightning_nodes)
nc = len(self.parent.network.channel_db)
np = len(self.parent.wallet.lnworker.peers)
self.status.setText(_('{} peers, {} nodes, {} channels').format(np, n, nc))
channel_db = self.parent.network.channel_db
num_nodes = len(channel_db.nodes)
num_channels = len(channel_db)
num_peers = len(self.parent.wallet.lnworker.peers)
self.status.setText(_('{} peers, {} nodes, {} channels')
.format(num_peers, num_nodes, num_channels))
def new_channel_dialog(self):
lnworker = self.parent.wallet.lnworker
@@ -116,15 +118,16 @@ class ChannelsList(MyTreeWidget):
peer = lnworker.peers.get(node_id)
if not peer:
known = node_id in self.parent.network.lightning_nodes
all_nodes = self.parent.network.channel_db.nodes
node_info = all_nodes.get(node_id, None)
if rest is not None:
try:
host, port = rest.split(":")
except ValueError:
self.parent.show_error(_('Connection strings must be in <node_pubkey>@<host>:<port> format'))
elif known:
node = self.network.lightning_nodes.get(node_id)
host, port = node['addresses'][0]
return
elif node_info:
host, port = node_info.addresses[0]
else:
self.parent.show_error(_('Unknown node:') + ' ' + nodeid_hex)
return

35
electrum/lnbase.py
View File

@@ -29,7 +29,7 @@ from . import crypto
from .crypto import sha256
from . import constants
from . import transaction
from .util import PrintError, bh2u, print_error, bfh, profiler, xor_bytes
from .util import PrintError, bh2u, print_error, bfh
from .transaction import opcodes, Transaction
from .lnonion import new_onion_packet, OnionHopsDataSingle, OnionPerHop, decode_onion_error
from .lnaddr import lndecode
@@ -428,38 +428,7 @@ class Peer(PrintError):
self.funding_signed[channel_id].put_nowait(payload)
def on_node_announcement(self, payload):
pubkey = payload['node_id']
signature = payload['signature']
h = bitcoin.Hash(payload['raw'][66:])
if not ecc.verify_signature(pubkey, signature, h):
return False
self.s = payload['addresses']
def read(n):
data, self.s = self.s[0:n], self.s[n:]
return data
addresses = []
while self.s:
atype = ord(read(1))
if atype == 0:
pass
elif atype == 1:
ipv4_addr = '.'.join(map(lambda x: '%d' % x, read(4)))
port = int.from_bytes(read(2), 'big')
x = ipv4_addr, port, binascii.hexlify(pubkey)
addresses.append((ipv4_addr, port))
elif atype == 2:
ipv6_addr = b':'.join([binascii.hexlify(read(2)) for i in range(4)])
port = int.from_bytes(read(2), 'big')
addresses.append((ipv6_addr, port))
else:
pass
continue
alias = payload['alias'].rstrip(b'\x00')
self.network.lightning_nodes[pubkey] = {
'alias': alias,
'addresses': addresses
}
#self.print_error('node announcement', binascii.hexlify(pubkey), alias, addresses)
self.channel_db.on_node_announcement(payload)
self.network.trigger_callback('ln_status')
def on_init(self, payload):

135
electrum/lnrouter.py
View File

@@ -29,17 +29,31 @@ import json
import threading
from collections import namedtuple, defaultdict
from typing import Sequence, Union, Tuple, Optional
import binascii
import base64
from . import constants
from .util import PrintError, bh2u, profiler, get_headers_dir, bfh
from .util import PrintError, bh2u, profiler, get_headers_dir, bfh, is_ip_address, list_enabled_bits
from .storage import JsonDB
from .lnchanannverifier import LNChanAnnVerifier, verify_sig_for_channel_update
from .crypto import Hash
from . import ecc
from .lnutil import LN_GLOBAL_FEATURE_BITS
class UnknownEvenFeatureBits(Exception): pass
class ChannelInfo(PrintError):
def __init__(self, channel_announcement_payload):
self.features_len = channel_announcement_payload['len']
self.features = channel_announcement_payload['features']
enabled_features = list_enabled_bits(int.from_bytes(self.features, "big"))
for fbit in enabled_features:
if fbit not in LN_GLOBAL_FEATURE_BITS and fbit % 2 == 0:
raise UnknownEvenFeatureBits()
self.channel_id = channel_announcement_payload['short_channel_id']
self.node_id_1 = channel_announcement_payload['node_id_1']
self.node_id_2 = channel_announcement_payload['node_id_2']
@@ -47,8 +61,6 @@ class ChannelInfo(PrintError):
assert type(self.node_id_2) is bytes
assert list(sorted([self.node_id_1, self.node_id_2])) == [self.node_id_1, self.node_id_2]
self.features_len = channel_announcement_payload['len']
self.features = channel_announcement_payload['features']
self.bitcoin_key_1 = channel_announcement_payload['bitcoin_key_1']
self.bitcoin_key_2 = channel_announcement_payload['bitcoin_key_2']
@@ -162,6 +174,86 @@ class ChannelInfoDirectedPolicy:
return ChannelInfoDirectedPolicy(d2)
class NodeInfo(PrintError):
def __init__(self, node_announcement_payload, addresses_already_parsed=False):
self.pubkey = node_announcement_payload['node_id']
self.features_len = node_announcement_payload['flen']
self.features = node_announcement_payload['features']
enabled_features = list_enabled_bits(int.from_bytes(self.features, "big"))
for fbit in enabled_features:
if fbit not in LN_GLOBAL_FEATURE_BITS and fbit % 2 == 0:
raise UnknownEvenFeatureBits()
if not addresses_already_parsed:
self.addresses = self.parse_addresses_field(node_announcement_payload['addresses'])
else:
self.addresses = node_announcement_payload['addresses']
self.alias = node_announcement_payload['alias'].rstrip(b'\x00')
self.timestamp = int.from_bytes(node_announcement_payload['timestamp'], "big")
@classmethod
def parse_addresses_field(cls, addresses_field):
buf = addresses_field
def read(n):
nonlocal buf
data, buf = buf[0:n], buf[n:]
return data
addresses = []
while buf:
atype = ord(read(1))
if atype == 0:
pass
elif atype == 1: # IPv4
ipv4_addr = '.'.join(map(lambda x: '%d' % x, read(4)))
port = int.from_bytes(read(2), 'big')
if is_ip_address(ipv4_addr) and port != 0:
addresses.append((ipv4_addr, port))
elif atype == 2: # IPv6
ipv6_addr = b':'.join([binascii.hexlify(read(2)) for i in range(8)])
ipv6_addr = ipv6_addr.decode('ascii')
port = int.from_bytes(read(2), 'big')
if is_ip_address(ipv6_addr) and port != 0:
addresses.append((ipv6_addr, port))
elif atype == 3: # onion v2
host = base64.b32encode(read(10)) + b'.onion'
host = host.decode('ascii').lower()
port = int.from_bytes(read(2), 'big')
addresses.append((host, port))
elif atype == 4: # onion v3
host = base64.b32encode(read(35)) + b'.onion'
host = host.decode('ascii').lower()
port = int.from_bytes(read(2), 'big')
addresses.append((host, port))
else:
# unknown address type
# we don't know how long it is -> have to escape
# if there are other addresses we could have parsed later, they are lost.
break
return addresses
def to_json(self) -> dict:
d = {}
d['node_id'] = bh2u(self.pubkey)
d['flen'] = bh2u(self.features_len)
d['features'] = bh2u(self.features)
d['addresses'] = self.addresses
d['alias'] = bh2u(self.alias)
d['timestamp'] = self.timestamp
return d
@classmethod
def from_json(cls, d: dict):
if d is None: return None
d2 = {}
d2['node_id'] = bfh(d['node_id'])
d2['flen'] = bfh(d['flen'])
d2['features'] = bfh(d['features'])
d2['addresses'] = d['addresses']
d2['alias'] = bfh(d['alias'])
d2['timestamp'] = d['timestamp'].to_bytes(4, "big")
return NodeInfo(d2, addresses_already_parsed=True)
class ChannelDB(JsonDB):
def __init__(self, network):
@@ -173,6 +265,7 @@ class ChannelDB(JsonDB):
self.lock = threading.Lock()
self._id_to_channel_info = {}
self._channels_for_node = defaultdict(set) # node -> set(short_channel_id)
self.nodes = {} # node_id -> NodeInfo
self.ca_verifier = LNChanAnnVerifier(network, self)
self.network.add_jobs([self.ca_verifier])
@@ -184,21 +277,35 @@ class ChannelDB(JsonDB):
with open(self.path, "r", encoding='utf-8') as f:
raw = f.read()
self.data = json.loads(raw)
# channels
channel_infos = self.get('channel_infos', {})
for short_channel_id, channel_info_d in channel_infos.items():
channel_info = ChannelInfo.from_json(channel_info_d)
short_channel_id = bfh(short_channel_id)
self.add_verified_channel_info(short_channel_id, channel_info)
# nodes
node_infos = self.get('node_infos', {})
for node_id, node_info_d in node_infos.items():
node_info = NodeInfo.from_json(node_info_d)
node_id = bfh(node_id)
self.nodes[node_id] = node_info
def save_data(self):
with self.lock:
# channels
channel_infos = {}
for short_channel_id, channel_info in self._id_to_channel_info.items():
channel_infos[bh2u(short_channel_id)] = channel_info
self.put('channel_infos', channel_infos)
# nodes
node_infos = {}
for node_id, node_info in self.nodes.items():
node_infos[bh2u(node_id)] = node_info
self.put('node_infos', node_infos)
self.write()
def __len__(self):
# number of channels
return len(self._id_to_channel_info)
def get_channel_info(self, channel_id) -> Optional[ChannelInfo]:
@@ -220,7 +327,10 @@ class ChannelDB(JsonDB):
return
if constants.net.rev_genesis_bytes() != msg_payload['chain_hash']:
return
channel_info = ChannelInfo(msg_payload)
try:
channel_info = ChannelInfo(msg_payload)
except UnknownEvenFeatureBits:
return
if trusted:
self.add_verified_channel_info(short_channel_id, channel_info)
else:
@@ -244,6 +354,21 @@ class ChannelDB(JsonDB):
return
channel_info.on_channel_update(msg_payload, trusted=trusted)
def on_node_announcement(self, msg_payload):
pubkey = msg_payload['node_id']
signature = msg_payload['signature']
h = Hash(msg_payload['raw'][66:])
if not ecc.verify_signature(pubkey, signature, h):
return
old_node_info = self.nodes.get(pubkey, None)
try:
new_node_info = NodeInfo(msg_payload)
except UnknownEvenFeatureBits:
return
if old_node_info and old_node_info.timestamp >= new_node_info.timestamp:
return # ignore
self.nodes[pubkey] = new_node_info
def remove_channel(self, short_channel_id):
try:
channel_info = self._id_to_channel_info[short_channel_id]

18
electrum/lnutil.py
View File

@@ -1,4 +1,4 @@
from .util import bfh, bh2u
from .util import bfh, bh2u, inv_dict
from .crypto import sha256
import json
from collections import namedtuple
@@ -380,3 +380,19 @@ def overall_weight(num_htlc):
def get_ecdh(priv: bytes, pub: bytes) -> bytes:
pt = ECPubkey(pub) * string_to_number(priv)
return sha256(pt.get_public_key_bytes())
LN_LOCAL_FEATURE_BITS = {
0: 'option_data_loss_protect_req',
1: 'option_data_loss_protect_opt',
3: 'initial_routing_sync',
4: 'option_upfront_shutdown_script_req',
5: 'option_upfront_shutdown_script_opt',
6: 'gossip_queries_req',
7: 'gossip_queries_opt',
}
LN_LOCAL_FEATURE_BITS_INV = inv_dict(LN_LOCAL_FEATURE_BITS)
LN_GLOBAL_FEATURE_BITS = {}
LN_GLOBAL_FEATURE_BITS_INV = inv_dict(LN_GLOBAL_FEATURE_BITS)

9
electrum/lnworker.py
View File

@@ -228,11 +228,12 @@ class LNWorker(PrintError):
self.peers.pop(k)
if len(self.peers) > 3:
continue
if not self.network.lightning_nodes:
if not self.network.channel_db.nodes:
continue
node_id = random.choice(list(self.network.lightning_nodes.keys()))
node = self.network.lightning_nodes.get(node_id)
addresses = node.get('addresses')
all_nodes = self.network.channel_db.nodes
node_id = random.choice(list(all_nodes))
node = all_nodes.get(node_id)
addresses = node.addresses
if addresses:
host, port = addresses[0]
self.print_error("trying node", bh2u(node_id))

1
electrum/network.py
View File

@@ -300,7 +300,6 @@ class Network(Logger):
self._set_status('disconnected')
# lightning network
self.lightning_nodes = {}
self.channel_db = lnrouter.ChannelDB(self)
self.path_finder = lnrouter.LNPathFinder(self.channel_db)
self.lnwatcher = lnwatcher.LNWatcher(self)

21
electrum/tests/test_util.py
View File

@@ -1,7 +1,7 @@
from decimal import Decimal
from electrum.util import (format_satoshis, format_fee_satoshis, parse_URI,
is_hash256_str, chunks)
is_hash256_str, chunks, is_ip_address, list_enabled_bits)
from . import SequentialTestCase
@@ -110,3 +110,22 @@ class TestUtil(SequentialTestCase):
list(chunks([1, 2, 3, 4, 5], 2)))
with self.assertRaises(ValueError):
list(chunks([1, 2, 3], 0))
def test_list_enabled_bits(self):
self.assertEqual((0, 2, 3, 6), list_enabled_bits(77))
self.assertEqual((), list_enabled_bits(0))
def test_is_ip_address(self):
self.assertTrue(is_ip_address("127.0.0.1"))
self.assertTrue(is_ip_address("127.000.000.1"))
self.assertTrue(is_ip_address("255.255.255.255"))
self.assertFalse(is_ip_address("255.255.256.255"))
self.assertFalse(is_ip_address("123.456.789.000"))
self.assertTrue(is_ip_address("2001:0db8:0000:0000:0000:ff00:0042:8329"))
self.assertTrue(is_ip_address("2001:db8:0:0:0:ff00:42:8329"))
self.assertTrue(is_ip_address("2001:db8::ff00:42:8329"))
self.assertFalse(is_ip_address("2001:::db8::ff00:42:8329"))
self.assertTrue(is_ip_address("::1"))
self.assertFalse(is_ip_address("2001:db8:0:0:g:ff00:42:8329"))
self.assertFalse(is_ip_address("lol"))
self.assertFalse(is_ip_address(":@ASD:@AS\x77\x22\xff¬!"))

20
electrum/util.py
View File

@@ -23,7 +23,7 @@
import binascii
import os, sys, re, json
from collections import defaultdict, OrderedDict
from typing import NamedTuple, Union, TYPE_CHECKING, Tuple, Optional, Callable, Any
from typing import NamedTuple, Union, TYPE_CHECKING, Tuple, Optional, Callable, Any, Sequence
from datetime import datetime
import decimal
from decimal import Decimal
@@ -40,6 +40,7 @@ import json
import time
from typing import NamedTuple, Optional
import ssl
import ipaddress
import aiohttp
from aiohttp_socks import SocksConnector, SocksVer
@@ -1156,3 +1157,20 @@ def multisig_type(wallet_type):
if match:
match = [int(x) for x in match.group(1, 2)]
return match
def is_ip_address(x: Union[str, bytes]) -> bool:
if isinstance(x, bytes):
x = x.decode("utf-8")
try:
ipaddress.ip_address(x)
return True
except ValueError:
return False
def list_enabled_bits(x: int) -> Sequence[int]:
"""e.g. 77 (0b1001101) --> (0, 2, 3, 6)"""
binary = bin(x)[2:]
rev_bin = reversed(binary)
return tuple(i for i, b in enumerate(rev_bin) if b == '1')