From e28836eb1d5c8811715b8ae634895e7ab8a0e32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Sun, 15 Jun 2025 20:09:49 +0200 Subject: [PATCH] address_synchronizer: add a cache in front of get_utxos() get_utxos() is called pretty often, both spuriously, and on focus change, on tab switch, &c. It blocks as it iterates, functionally, /every/ address the wallet knows of. On large wallets (like testnet vpub5VfkVzoT7qgd5gUKjxgGE2oMJU4zKSktusfLx2NaQCTfSeeSY3S723qXKUZZaJzaF6YaF8nwQgbMTWx54Ugkf4NZvSxdzicENHoLJh96EKg from #6625 with 11k TXes and 10.5k addresses), this takes 1.3s of 100%ed CPU usage, basically in a loop from the UI thread. get_utxos() is 50-70% of the flame-graph when sampling a synced wallet process. This data is a function of the block-chain state, and we have hooks that notify us of when the block-chain state changes: we can just cache the result and only re-compute it then. For example, here's a trace log where get_utxos() has print(end - start, len(domain), block_height) and a transaction is clearing: 1.3775344607420266 10540 4507192 0.0010390589013695717 10540 4507192 cached! 0.001393263228237629 10540 4507192 cached! 0.0009001069702208042 10540 4507192 cached! 0.0010241391137242317 10540 4507192 cached! ... 0.00207632128149271 10540 4507192 cached! 0.001397700048983097 10540 4507192 cached! invalidate_cache 1.4686454269103706 10540 4507192 0.0012429207563400269 10540 4507192 cached! 0.0015075239352881908 10540 4507192 cached! 0.0010459059849381447 10540 4507192 cached! 0.0009669591672718525 10540 4507192 cached! ... on_event_blockchain_updated invalidate_cache 1.3897203942760825 10540 4507193 0.0010689008049666882 10540 4507193 cached! 0.0010420521721243858 10540 4507193 cached! ... invalidate_cache 1.408584670163691 10540 4507193 0.001336586195975542 10540 4507193 cached! 0.0009196233004331589 10540 4507193 cached! 0.0009176661260426044 10540 4507193 cached! ... about 30s of low activity. Without this patch, the UI is prone to freezing, running behind, and I wouldn't be surprised if UI thread blocking on Windows ends up crashing to some extent as the issue notes. In the log, this manifests as a much slower but consistent stream of full 1.3-1.4s updates during use, and every time the window is focused. --- electrum/address_synchronizer.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py index d75ae6156..3d6cc367a 100644 --- a/electrum/address_synchronizer.py +++ b/electrum/address_synchronizer.py @@ -22,6 +22,7 @@ # SOFTWARE. import asyncio +import copy import threading import itertools from collections import defaultdict @@ -99,9 +100,15 @@ class AddressSynchronizer(Logger, EventListener): self.threadlocal_cache = threading.local() self._get_balance_cache = {} + self._get_utxos_cache = {} self.load_and_cleanup() + @with_lock + def invalidate_cache(self): + self._get_balance_cache.clear() + self._get_utxos_cache.clear() + def diagnostic_name(self): return self.name or "" @@ -203,7 +210,7 @@ class AddressSynchronizer(Logger, EventListener): @event_listener @with_lock def on_event_blockchain_updated(self, *args): - self._get_balance_cache = {} # invalidate cache + self.invalidate_cache() self.db.put('stored_height', self.get_local_height()) async def stop(self): @@ -335,7 +342,7 @@ class AddressSynchronizer(Logger, EventListener): pass else: self.db.add_txi_addr(tx_hash, addr, ser, v) - self._get_balance_cache.clear() # invalidate cache + self.invalidate_cache() for txi in tx.inputs(): if txi.is_coinbase_input(): continue @@ -353,7 +360,7 @@ class AddressSynchronizer(Logger, EventListener): addr = txo.address if addr and self.is_mine(addr): self.db.add_txo_addr(tx_hash, addr, n, v, is_coinbase) - self._get_balance_cache.clear() # invalidate cache + self.invalidate_cache() # give v to txi that spends me next_tx = self.db.get_spent_outpoint(tx_hash, n) if next_tx is not None: @@ -405,7 +412,7 @@ class AddressSynchronizer(Logger, EventListener): remove_from_spent_outpoints() self._remove_tx_from_local_history(tx_hash) for addr in itertools.chain(self.db.get_txi_addresses(tx_hash), self.db.get_txo_addresses(tx_hash)): - self._get_balance_cache.clear() # invalidate cache + self.invalidate_cache() self.db.remove_txi(tx_hash) self.db.remove_txo(tx_hash) self.db.remove_tx_fee(tx_hash) @@ -503,7 +510,7 @@ class AddressSynchronizer(Logger, EventListener): def clear_history(self): self.db.clear_history() self._history_local.clear() - self._get_balance_cache.clear() # invalidate cache + self.invalidate_cache() @with_lock def _get_tx_sort_key(self, tx_hash: str) -> Tuple[int, int]: @@ -970,6 +977,13 @@ class AddressSynchronizer(Logger, EventListener): if excluded_addresses: domain = set(domain) - set(excluded_addresses) mempool_height = block_height + 1 # height of next block + cache_key = sha256( + ','.join(sorted(domain)) + + f";{mature_only};{confirmed_funding_only};{confirmed_spending_only};{nonlocal_only};{block_height}" + ) + cached = self._get_utxos_cache.get(cache_key) + if cached is not None: + return copy.deepcopy(cached) for addr in domain: txos = self.get_addr_outputs(addr) for txo in txos.values(): @@ -987,6 +1001,7 @@ class AddressSynchronizer(Logger, EventListener): continue coins.append(txo) continue + self._get_utxos_cache[cache_key] = copy.deepcopy(coins) return coins def is_used(self, address: str) -> bool: