423 lines
13 KiB
Python
423 lines
13 KiB
Python
#!/usr/bin/env python
|
|
#
|
|
# Electrum - lightweight Bitcoin client
|
|
# Copyright (C) 2019 The Electrum Developers
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person
|
|
# obtaining a copy of this software and associated documentation files
|
|
# (the "Software"), to deal in the Software without restriction,
|
|
# including without limitation the rights to use, copy, modify, merge,
|
|
# publish, distribute, sublicense, and/or sell copies of the Software,
|
|
# and to permit persons to whom the Software is furnished to do so,
|
|
# subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be
|
|
# included in all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
|
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
|
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
import threading
|
|
import copy
|
|
import json
|
|
from typing import TYPE_CHECKING, Optional
|
|
import jsonpatch
|
|
|
|
from . import util
|
|
from .util import WalletFileException, profiler
|
|
from .logging import Logger
|
|
|
|
if TYPE_CHECKING:
|
|
from .storage import WalletStorage
|
|
|
|
def modifier(func):
|
|
def wrapper(self, *args, **kwargs):
|
|
with self.lock:
|
|
self._modified = True
|
|
return func(self, *args, **kwargs)
|
|
return wrapper
|
|
|
|
def locked(func):
|
|
def wrapper(self, *args, **kwargs):
|
|
with self.lock:
|
|
return func(self, *args, **kwargs)
|
|
return wrapper
|
|
|
|
|
|
registered_names = {}
|
|
registered_dicts = {}
|
|
registered_dict_keys = {}
|
|
registered_parent_keys = {}
|
|
|
|
def register_dict(name, method, _type):
|
|
registered_dicts[name] = method, _type
|
|
|
|
def register_name(name, method, _type):
|
|
registered_names[name] = method, _type
|
|
|
|
def register_dict_key(name, method):
|
|
registered_dict_keys[name] = method
|
|
|
|
def register_parent_key(name, method):
|
|
registered_parent_keys[name] = method
|
|
|
|
def stored_as(name, _type=dict):
|
|
""" decorator that indicates the storage key of a stored object"""
|
|
def decorator(func):
|
|
registered_names[name] = func, _type
|
|
return func
|
|
return decorator
|
|
|
|
def stored_in(name, _type=dict):
|
|
""" decorator that indicates the storage key of an element in a StoredDict"""
|
|
def decorator(func):
|
|
registered_dicts[name] = func, _type
|
|
return func
|
|
return decorator
|
|
|
|
|
|
def key_path(path, key):
|
|
def to_str(x):
|
|
if isinstance(x, int):
|
|
return str(int(x))
|
|
else:
|
|
assert isinstance(x, str)
|
|
return x
|
|
return '/' + '/'.join([to_str(x) for x in path + [to_str(key)]])
|
|
|
|
|
|
class StoredObject:
|
|
|
|
db = None
|
|
path = None
|
|
|
|
def __setattr__(self, key, value):
|
|
if self.db and key not in ['path', 'db'] and not key.startswith('_'):
|
|
if value != getattr(self, key):
|
|
self.db.add_patch({'op': 'replace', 'path': key_path(self.path, key), 'value': value})
|
|
object.__setattr__(self, key, value)
|
|
|
|
def set_db(self, db, path):
|
|
self.db = db
|
|
self.path = path
|
|
|
|
def to_json(self):
|
|
d = dict(vars(self))
|
|
d.pop('db', None)
|
|
d.pop('path', None)
|
|
# don't expose/store private stuff
|
|
d = {k: v for k, v in d.items()
|
|
if not k.startswith('_')}
|
|
return d
|
|
|
|
|
|
_RaiseKeyError = object() # singleton for no-default behavior
|
|
|
|
class StoredDict(dict):
|
|
|
|
def __init__(self, data, db, path):
|
|
self.db = db
|
|
self.lock = self.db.lock if self.db else threading.RLock()
|
|
self.path = path
|
|
# recursively convert dicts to StoredDict
|
|
for k, v in list(data.items()):
|
|
self.__setitem__(k, v, patch=False)
|
|
|
|
@locked
|
|
def __setitem__(self, key, v, patch=True):
|
|
is_new = key not in self
|
|
# early return to prevent unnecessary disk writes
|
|
if not is_new and patch:
|
|
if self.db and json.dumps(v, cls=self.db.encoder) == json.dumps(self[key], cls=self.db.encoder):
|
|
return
|
|
# recursively set db and path
|
|
if isinstance(v, StoredDict):
|
|
#assert v.db is None
|
|
v.db = self.db
|
|
v.path = self.path + [key]
|
|
for k, vv in v.items():
|
|
v.__setitem__(k, vv, patch=False)
|
|
# recursively convert dict to StoredDict.
|
|
# _convert_dict is called breadth-first
|
|
elif isinstance(v, dict):
|
|
if self.db:
|
|
v = self.db._convert_dict(self.path, key, v)
|
|
if not self.db or self.db._should_convert_to_stored_dict(key):
|
|
v = StoredDict(v, self.db, self.path + [key])
|
|
# convert_value is called depth-first
|
|
if isinstance(v, dict) or isinstance(v, str) or isinstance(v, int):
|
|
if self.db:
|
|
v = self.db._convert_value(self.path, key, v)
|
|
# set parent of StoredObject
|
|
if isinstance(v, StoredObject):
|
|
v.set_db(self.db, self.path + [key])
|
|
# convert lists
|
|
if isinstance(v, list):
|
|
v = StoredList(v, self.db, self.path + [key])
|
|
# reject sets. they do not work well with jsonpatch
|
|
if isinstance(v, set):
|
|
raise Exception(f"Do not store sets inside jsondb. path={self.path!r}")
|
|
# set item
|
|
dict.__setitem__(self, key, v)
|
|
if self.db and patch:
|
|
op = 'add' if is_new else 'replace'
|
|
self.db.add_patch({'op': op, 'path': key_path(self.path, key), 'value': v})
|
|
|
|
@locked
|
|
def __delitem__(self, key):
|
|
dict.__delitem__(self, key)
|
|
if self.db:
|
|
self.db.add_patch({'op': 'remove', 'path': key_path(self.path, key)})
|
|
|
|
@locked
|
|
def pop(self, key, v=_RaiseKeyError):
|
|
if key not in self:
|
|
if v is _RaiseKeyError:
|
|
raise KeyError(key)
|
|
else:
|
|
return v
|
|
r = dict.pop(self, key)
|
|
if self.db:
|
|
self.db.add_patch({'op': 'remove', 'path': key_path(self.path, key)})
|
|
return r
|
|
|
|
|
|
class StoredList(list):
|
|
|
|
def __init__(self, data, db, path):
|
|
list.__init__(self, data)
|
|
self.db = db
|
|
self.lock = self.db.lock if self.db else threading.RLock()
|
|
self.path = path
|
|
|
|
@locked
|
|
def append(self, item):
|
|
n = len(self)
|
|
list.append(self, item)
|
|
if self.db:
|
|
self.db.add_patch({'op': 'add', 'path': key_path(self.path, '%d'%n), 'value':item})
|
|
|
|
@locked
|
|
def remove(self, item):
|
|
n = self.index(item)
|
|
list.remove(self, item)
|
|
if self.db:
|
|
self.db.add_patch({'op': 'remove', 'path': key_path(self.path, '%d'%n)})
|
|
|
|
|
|
|
|
class JsonDB(Logger):
|
|
|
|
def __init__(
|
|
self,
|
|
s: str,
|
|
*,
|
|
storage: Optional['WalletStorage'] = None,
|
|
encoder=None,
|
|
upgrader=None,
|
|
):
|
|
Logger.__init__(self)
|
|
self.lock = threading.RLock()
|
|
self.storage = storage
|
|
self.encoder = encoder
|
|
self.pending_changes = []
|
|
self._modified = False
|
|
# load data
|
|
data = self.load_data(s)
|
|
if upgrader:
|
|
data, was_upgraded = upgrader(data)
|
|
self._modified |= was_upgraded
|
|
# convert to StoredDict
|
|
self.data = StoredDict(data, self, [])
|
|
# write file in case there was a db upgrade
|
|
if self.storage and self.storage.file_exists():
|
|
self.write_and_force_consolidation()
|
|
|
|
def load_data(self, s: str) -> dict:
|
|
if s == '':
|
|
return {}
|
|
try:
|
|
data = json.loads('[' + s + ']')
|
|
data, patches = data[0], data[1:]
|
|
except Exception:
|
|
if r := self.maybe_load_ast_data(s):
|
|
data, patches = r, []
|
|
elif r := self.maybe_load_incomplete_data(s):
|
|
data, patches = r, []
|
|
else:
|
|
raise WalletFileException("Cannot read wallet file. (parsing failed)")
|
|
if not isinstance(data, dict):
|
|
raise WalletFileException("Malformed wallet file (not dict)")
|
|
if patches:
|
|
# apply patches
|
|
self.logger.info('found %d patches'%len(patches))
|
|
patch = jsonpatch.JsonPatch(patches)
|
|
data = patch.apply(data)
|
|
self.set_modified(True)
|
|
return data
|
|
|
|
def maybe_load_ast_data(self, s):
|
|
""" for old wallets """
|
|
try:
|
|
import ast
|
|
d = ast.literal_eval(s)
|
|
labels = d.get('labels', {})
|
|
except Exception as e:
|
|
return
|
|
data = {}
|
|
for key, value in d.items():
|
|
try:
|
|
json.dumps(key)
|
|
json.dumps(value)
|
|
except Exception:
|
|
self.logger.info(f'Failed to convert label to json format: {key}')
|
|
continue
|
|
data[key] = value
|
|
return data
|
|
|
|
def maybe_load_incomplete_data(self, s):
|
|
n = s.count('{') - s.count('}')
|
|
i = len(s)
|
|
while n > 0 and i > 0:
|
|
i = i - 1
|
|
if s[i] == '{':
|
|
n = n - 1
|
|
if s[i] == '}':
|
|
n = n + 1
|
|
if n == 0:
|
|
s = s[0:i]
|
|
assert s[-2:] == ',\n'
|
|
self.logger.info('found incomplete data {s[i:]}')
|
|
return self.load_data(s[0:-2])
|
|
|
|
def set_modified(self, b):
|
|
with self.lock:
|
|
self._modified = b
|
|
|
|
def modified(self):
|
|
return self._modified
|
|
|
|
@locked
|
|
def add_patch(self, patch):
|
|
self.pending_changes.append(json.dumps(patch, cls=self.encoder))
|
|
self.set_modified(True)
|
|
|
|
@locked
|
|
def get(self, key, default=None):
|
|
v = self.data.get(key)
|
|
if v is None:
|
|
v = default
|
|
return v
|
|
|
|
@modifier
|
|
def put(self, key, value):
|
|
try:
|
|
json.dumps(key, cls=self.encoder)
|
|
json.dumps(value, cls=self.encoder)
|
|
except Exception:
|
|
self.logger.info(f"json error: cannot save {repr(key)} ({repr(value)})")
|
|
return False
|
|
if value is not None:
|
|
if self.data.get(key) != value:
|
|
self.data[key] = copy.deepcopy(value)
|
|
return True
|
|
elif key in self.data:
|
|
self.data.pop(key)
|
|
return True
|
|
return False
|
|
|
|
@locked
|
|
def get_dict(self, name) -> dict:
|
|
# Warning: interacts un-intuitively with 'put': certain parts
|
|
# of 'data' will have pointers saved as separate variables.
|
|
if name not in self.data:
|
|
self.data[name] = {}
|
|
return self.data[name]
|
|
|
|
@locked
|
|
def get_stored_item(self, key, default) -> dict:
|
|
if key not in self.data:
|
|
self.data[key] = default
|
|
return self.data[key]
|
|
|
|
@locked
|
|
def dump(self, *, human_readable: bool = True) -> str:
|
|
"""Serializes the DB as a string.
|
|
'human_readable': makes the json indented and sorted, but this is ~2x slower
|
|
"""
|
|
return json.dumps(
|
|
self.data,
|
|
indent=4 if human_readable else None,
|
|
sort_keys=bool(human_readable),
|
|
cls=self.encoder,
|
|
)
|
|
|
|
def _should_convert_to_stored_dict(self, key) -> bool:
|
|
return True
|
|
|
|
def _convert_dict(self, path, key, v):
|
|
if key in registered_dicts:
|
|
constructor, _type = registered_dicts[key]
|
|
if _type == dict:
|
|
v = dict((k, constructor(**x)) for k, x in v.items())
|
|
elif _type == tuple:
|
|
v = dict((k, constructor(*x)) for k, x in v.items())
|
|
else:
|
|
v = dict((k, constructor(x)) for k, x in v.items())
|
|
if key in registered_dict_keys:
|
|
convert_key = registered_dict_keys[key]
|
|
elif path and path[-1] in registered_parent_keys:
|
|
convert_key = registered_parent_keys.get(path[-1])
|
|
else:
|
|
convert_key = None
|
|
if convert_key:
|
|
v = dict((convert_key(k), x) for k, x in v.items())
|
|
return v
|
|
|
|
def _convert_value(self, path, key, v):
|
|
if key in registered_names:
|
|
constructor, _type = registered_names[key]
|
|
if _type == dict:
|
|
v = constructor(**v)
|
|
else:
|
|
v = constructor(v)
|
|
return v
|
|
|
|
@locked
|
|
def write(self):
|
|
if (not self.storage.file_exists()
|
|
or self.storage.is_encrypted()
|
|
or self.storage.needs_consolidation()):
|
|
self.write_and_force_consolidation()
|
|
else:
|
|
self._append_pending_changes()
|
|
|
|
@locked
|
|
def _append_pending_changes(self):
|
|
if threading.current_thread().daemon:
|
|
raise Exception('daemon thread cannot write db')
|
|
if not self.pending_changes:
|
|
self.logger.info('no pending changes')
|
|
return
|
|
self.logger.info(f'appending {len(self.pending_changes)} pending changes')
|
|
s = ''.join([',\n' + x for x in self.pending_changes])
|
|
self.storage.append(s)
|
|
self.pending_changes = []
|
|
|
|
@locked
|
|
@profiler
|
|
def write_and_force_consolidation(self):
|
|
if threading.current_thread().daemon:
|
|
raise Exception('daemon thread cannot write db')
|
|
if not self.modified():
|
|
return
|
|
json_str = self.dump(human_readable=not self.storage.is_encrypted())
|
|
self.storage.write(json_str)
|
|
self.pending_changes = []
|
|
self.set_modified(False)
|