From 8b99c218b455900d87f27a80a89f5cca6852a398 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 6 Aug 2025 12:59:05 +0000 Subject: [PATCH 1/8] network: move broadcast_transaction move network to interface.py --- electrum/interface.py | 239 +++++++++++++++++++++++++++++++++++++++++- electrum/network.py | 238 +---------------------------------------- 2 files changed, 241 insertions(+), 236 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index e47edd024..c9f438e47 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -49,7 +49,8 @@ import certifi from .util import (ignore_exceptions, log_exceptions, bfh, ESocksProxy, is_integer, is_non_negative_integer, is_hash256_str, is_hex_str, - is_int_or_float, is_non_negative_int_or_float, OldTaskGroup) + is_int_or_float, is_non_negative_int_or_float, OldTaskGroup, + send_exception_to_crash_reporter, error_text_str_to_safe_str) from . import util from . import x509 from . import pem @@ -57,6 +58,7 @@ from . import version from . import blockchain from .blockchain import Blockchain, HEADER_SIZE, CHUNK_SIZE from . import bitcoin +from .bitcoin import DummyAddress, DummyAddressUsedInTxException from . import constants from .i18n import _ from .logging import Logger @@ -279,6 +281,34 @@ class InvalidOptionCombination(Exception): pass class ConnectError(NetworkException): pass +class TxBroadcastError(NetworkException): + def get_message_for_gui(self): + raise NotImplementedError() + + +class TxBroadcastHashMismatch(TxBroadcastError): + def get_message_for_gui(self): + return "{}\n{}\n\n{}" \ + .format(_("The server returned an unexpected transaction ID when broadcasting the transaction."), + _("Consider trying to connect to a different server, or updating Electrum."), + str(self)) + + +class TxBroadcastServerReturnedError(TxBroadcastError): + def get_message_for_gui(self): + return "{}\n{}\n\n{}" \ + .format(_("The server returned an error when broadcasting the transaction."), + _("Consider trying to connect to a different server, or updating Electrum."), + str(self)) + + +class TxBroadcastUnknownError(TxBroadcastError): + def get_message_for_gui(self): + return "{}\n{}" \ + .format(_("Unknown error when broadcasting the transaction."), + _("Consider trying to connect to a different server, or updating Electrum.")) + + class _RSClient(RSClient): async def create_connection(self): try: @@ -1296,6 +1326,29 @@ class Interface(Logger): raise RequestCorrupted(f"received tx does not match expected txid {tx_hash} (got {tx.txid()})") return raw + async def broadcast_transaction(self, tx: 'Transaction', *, timeout=None) -> None: + """caller should handle TxBroadcastError and RequestTimedOut""" + if timeout is None: + timeout = self.network.get_network_timeout_seconds(NetworkTimeout.Urgent) + if any(DummyAddress.is_dummy_address(txout.address) for txout in tx.outputs()): + raise DummyAddressUsedInTxException("tried to broadcast tx with dummy address!") + try: + out = await self.session.send_request('blockchain.transaction.broadcast', [tx.serialize()], timeout=timeout) + # note: both 'out' and exception messages are untrusted input from the server + except (RequestTimedOut, asyncio.CancelledError, asyncio.TimeoutError): + raise # pass-through + except aiorpcx.jsonrpc.CodeMessageError as e: + self.logger.info(f"broadcast_transaction error [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") + raise TxBroadcastServerReturnedError(sanitize_tx_broadcast_response(e.message)) from e + except BaseException as e: # intentional BaseException for sanity! + self.logger.info(f"broadcast_transaction error2 [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") + send_exception_to_crash_reporter(e) + raise TxBroadcastUnknownError() from e + if out != tx.txid(): + self.logger.info(f"unexpected txid for broadcast_transaction [DO NOT TRUST THIS MESSAGE]: " + f"{error_text_str_to_safe_str(out)} != {tx.txid()}. tx={str(tx)}") + raise TxBroadcastHashMismatch(_("Server returned unexpected transaction ID.")) + async def get_history_for_scripthash(self, sh: str) -> List[dict]: if not is_hash256_str(sh): raise Exception(f"{repr(sh)} is not a scripthash") @@ -1462,6 +1515,190 @@ def _assert_header_does_not_check_against_any_chain(header: dict) -> None: raise Exception('bad_header must not check!') +def sanitize_tx_broadcast_response(server_msg) -> str: + # Unfortunately, bitcoind and hence the Electrum protocol doesn't return a useful error code. + # So, we use substring matching to grok the error message. + # server_msg is untrusted input so it should not be shown to the user. see #4968 + server_msg = str(server_msg) + server_msg = server_msg.replace("\n", r"\n") + + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/script/script_error.cpp + script_error_messages = { + r"Script evaluated without error but finished with a false/empty top stack element", + r"Script failed an OP_VERIFY operation", + r"Script failed an OP_EQUALVERIFY operation", + r"Script failed an OP_CHECKMULTISIGVERIFY operation", + r"Script failed an OP_CHECKSIGVERIFY operation", + r"Script failed an OP_NUMEQUALVERIFY operation", + r"Script is too big", + r"Push value size limit exceeded", + r"Operation limit exceeded", + r"Stack size limit exceeded", + r"Signature count negative or greater than pubkey count", + r"Pubkey count negative or limit exceeded", + r"Opcode missing or not understood", + r"Attempted to use a disabled opcode", + r"Operation not valid with the current stack size", + r"Operation not valid with the current altstack size", + r"OP_RETURN was encountered", + r"Invalid OP_IF construction", + r"Negative locktime", + r"Locktime requirement not satisfied", + r"Signature hash type missing or not understood", + r"Non-canonical DER signature", + r"Data push larger than necessary", + r"Only push operators allowed in signatures", + r"Non-canonical signature: S value is unnecessarily high", + r"Dummy CHECKMULTISIG argument must be zero", + r"OP_IF/NOTIF argument must be minimal", + r"Signature must be zero for failed CHECK(MULTI)SIG operation", + r"NOPx reserved for soft-fork upgrades", + r"Witness version reserved for soft-fork upgrades", + r"Taproot version reserved for soft-fork upgrades", + r"OP_SUCCESSx reserved for soft-fork upgrades", + r"Public key version reserved for soft-fork upgrades", + r"Public key is neither compressed or uncompressed", + r"Stack size must be exactly one after execution", + r"Extra items left on stack after execution", + r"Witness program has incorrect length", + r"Witness program was passed an empty witness", + r"Witness program hash mismatch", + r"Witness requires empty scriptSig", + r"Witness requires only-redeemscript scriptSig", + r"Witness provided for non-witness script", + r"Using non-compressed keys in segwit", + r"Invalid Schnorr signature size", + r"Invalid Schnorr signature hash type", + r"Invalid Schnorr signature", + r"Invalid Taproot control block size", + r"Too much signature validation relative to witness weight", + r"OP_CHECKMULTISIG(VERIFY) is not available in tapscript", + r"OP_IF/NOTIF argument must be minimal in tapscript", + r"Using OP_CODESEPARATOR in non-witness script", + r"Signature is found in scriptCode", + } + for substring in script_error_messages: + if substring in server_msg: + return substring + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/validation.cpp + # grep "REJECT_" + # grep "TxValidationResult" + # should come after script_error.cpp (due to e.g. "non-mandatory-script-verify-flag") + validation_error_messages = { + r"coinbase": None, + r"tx-size-small": None, + r"non-final": None, + r"txn-already-in-mempool": None, + r"txn-mempool-conflict": None, + r"txn-already-known": None, + r"non-BIP68-final": None, + r"bad-txns-nonstandard-inputs": None, + r"bad-witness-nonstandard": None, + r"bad-txns-too-many-sigops": None, + r"mempool min fee not met": + ("mempool min fee not met\n" + + _("Your transaction is paying a fee that is so low that the bitcoin node cannot " + "fit it into its mempool. The mempool is already full of hundreds of megabytes " + "of transactions that all pay higher fees. Try to increase the fee.")), + r"min relay fee not met": None, + r"absurdly-high-fee": None, + r"max-fee-exceeded": None, + r"too-long-mempool-chain": None, + r"bad-txns-spends-conflicting-tx": None, + r"insufficient fee": ("insufficient fee\n" + + _("Your transaction is trying to replace another one in the mempool but it " + "does not meet the rules to do so. Try to increase the fee.")), + r"too many potential replacements": None, + r"replacement-adds-unconfirmed": None, + r"mempool full": None, + r"non-mandatory-script-verify-flag": None, + r"mandatory-script-verify-flag-failed": None, + r"Transaction check failed": None, + } + for substring in validation_error_messages: + if substring in server_msg: + msg = validation_error_messages[substring] + return msg if msg else substring + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/rpc/rawtransaction.cpp + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/util/error.cpp + # https://github.com/bitcoin/bitcoin/blob/3f83c744ac28b700090e15b5dda2260724a56f49/src/common/messages.cpp#L126 + # grep "RPC_TRANSACTION" + # grep "RPC_DESERIALIZATION_ERROR" + # grep "TransactionError" + rawtransaction_error_messages = { + r"Missing inputs": None, + r"Inputs missing or spent": None, + r"transaction already in block chain": None, + r"Transaction already in block chain": None, + r"Transaction outputs already in utxo set": None, + r"TX decode failed": None, + r"Peer-to-peer functionality missing or disabled": None, + r"Transaction rejected by AcceptToMemoryPool": None, + r"AcceptToMemoryPool failed": None, + r"Transaction rejected by mempool": None, + r"Mempool internal error": None, + r"Fee exceeds maximum configured by user": None, + r"Unspendable output exceeds maximum configured by user": None, + r"Transaction rejected due to invalid package": None, + } + for substring in rawtransaction_error_messages: + if substring in server_msg: + msg = rawtransaction_error_messages[substring] + return msg if msg else substring + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/consensus/tx_verify.cpp + # https://github.com/bitcoin/bitcoin/blob/c7ad94428ab6f54661d7a5441e1fdd0ebf034903/src/consensus/tx_check.cpp + # grep "REJECT_" + # grep "TxValidationResult" + tx_verify_error_messages = { + r"bad-txns-vin-empty": None, + r"bad-txns-vout-empty": None, + r"bad-txns-oversize": None, + r"bad-txns-vout-negative": None, + r"bad-txns-vout-toolarge": None, + r"bad-txns-txouttotal-toolarge": None, + r"bad-txns-inputs-duplicate": None, + r"bad-cb-length": None, + r"bad-txns-prevout-null": None, + r"bad-txns-inputs-missingorspent": + ("bad-txns-inputs-missingorspent\n" + + _("You might have a local transaction in your wallet that this transaction " + "builds on top. You need to either broadcast or remove the local tx.")), + r"bad-txns-premature-spend-of-coinbase": None, + r"bad-txns-inputvalues-outofrange": None, + r"bad-txns-in-belowout": None, + r"bad-txns-fee-outofrange": None, + } + for substring in tx_verify_error_messages: + if substring in server_msg: + msg = tx_verify_error_messages[substring] + return msg if msg else substring + # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/policy/policy.cpp + # grep "reason =" + # should come after validation.cpp (due to "tx-size" vs "tx-size-small") + # should come after script_error.cpp (due to e.g. "version") + policy_error_messages = { + r"version": _("Transaction uses non-standard version."), + r"tx-size": _("The transaction was rejected because it is too large (in bytes)."), + r"scriptsig-size": None, + r"scriptsig-not-pushonly": None, + r"scriptpubkey": + ("scriptpubkey\n" + + _("Some of the outputs pay to a non-standard script.")), + r"bare-multisig": None, + r"dust": + (_("Transaction could not be broadcast due to dust outputs.\n" + "Some of the outputs are too small in value, probably lower than 1000 satoshis.\n" + "Check the units, make sure you haven't confused e.g. mBTC and BTC.")), + r"multi-op-return": _("The transaction was rejected because it contains multiple OP_RETURN outputs."), + } + for substring in policy_error_messages: + if substring in server_msg: + msg = policy_error_messages[substring] + return msg if msg else substring + # otherwise: + return _("Unknown error") + + def check_cert(host, cert): try: b = pem.dePem(cert, 'CERTIFICATE') diff --git a/electrum/network.py b/electrum/network.py index 462cdaa74..b9a9bb7a7 100644 --- a/electrum/network.py +++ b/electrum/network.py @@ -43,10 +43,9 @@ from aiohttp import ClientResponse from . import util from .util import ( - log_exceptions, ignore_exceptions, OldTaskGroup, make_aiohttp_session, send_exception_to_crash_reporter, MyEncoder, + log_exceptions, ignore_exceptions, OldTaskGroup, make_aiohttp_session, MyEncoder, NetworkRetryManager, error_text_str_to_safe_str, detect_tor_socks_proxy ) -from .bitcoin import DummyAddress, DummyAddressUsedInTxException from . import constants from . import blockchain from . import dns_hacks @@ -54,7 +53,7 @@ from .transaction import Transaction from .blockchain import Blockchain from .interface import ( Interface, PREFERRED_NETWORK_PROTOCOL, RequestTimedOut, NetworkTimeout, BUCKET_NAME_OF_ONION_SERVERS, - NetworkException, RequestCorrupted, ServerAddr + NetworkException, RequestCorrupted, ServerAddr, TxBroadcastError, ) from .version import PROTOCOL_VERSION from .i18n import _ @@ -287,34 +286,6 @@ class NetworkParameters(NamedTuple): class BestEffortRequestFailed(NetworkException): pass -class TxBroadcastError(NetworkException): - def get_message_for_gui(self): - raise NotImplementedError() - - -class TxBroadcastHashMismatch(TxBroadcastError): - def get_message_for_gui(self): - return "{}\n{}\n\n{}" \ - .format(_("The server returned an unexpected transaction ID when broadcasting the transaction."), - _("Consider trying to connect to a different server, or updating Electrum."), - str(self)) - - -class TxBroadcastServerReturnedError(TxBroadcastError): - def get_message_for_gui(self): - return "{}\n{}\n\n{}" \ - .format(_("The server returned an error when broadcasting the transaction."), - _("Consider trying to connect to a different server, or updating Electrum."), - str(self)) - - -class TxBroadcastUnknownError(TxBroadcastError): - def get_message_for_gui(self): - return "{}\n{}" \ - .format(_("Unknown error when broadcasting the transaction."), - _("Consider trying to connect to a different server, or updating Electrum.")) - - class UntrustedServerReturnedError(NetworkException): def __init__(self, *, original_exception): self.original_exception = original_exception @@ -1096,26 +1067,7 @@ class Network(Logger, NetworkRetryManager[ServerAddr]): """caller should handle TxBroadcastError""" if self.interface is None: # handled by best_effort_reliable raise RequestTimedOut() - if timeout is None: - timeout = self.get_network_timeout_seconds(NetworkTimeout.Urgent) - if any(DummyAddress.is_dummy_address(txout.address) for txout in tx.outputs()): - raise DummyAddressUsedInTxException("tried to broadcast tx with dummy address!") - try: - out = await self.interface.session.send_request('blockchain.transaction.broadcast', [tx.serialize()], timeout=timeout) - # note: both 'out' and exception messages are untrusted input from the server - except (RequestTimedOut, asyncio.CancelledError, asyncio.TimeoutError): - raise # pass-through - except aiorpcx.jsonrpc.CodeMessageError as e: - self.logger.info(f"broadcast_transaction error [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") - raise TxBroadcastServerReturnedError(self.sanitize_tx_broadcast_response(e.message)) from e - except BaseException as e: # intentional BaseException for sanity! - self.logger.info(f"broadcast_transaction error2 [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") - send_exception_to_crash_reporter(e) - raise TxBroadcastUnknownError() from e - if out != tx.txid(): - self.logger.info(f"unexpected txid for broadcast_transaction [DO NOT TRUST THIS MESSAGE]: " - f"{error_text_str_to_safe_str(out)} != {tx.txid()}. tx={str(tx)}") - raise TxBroadcastHashMismatch(_("Server returned unexpected transaction ID.")) + await self.interface.broadcast_transaction(tx, timeout=timeout) async def try_broadcasting(self, tx, name) -> bool: try: @@ -1127,190 +1079,6 @@ class Network(Logger, NetworkRetryManager[ServerAddr]): self.logger.info(f'success: broadcasting {name} {tx.txid()}') return True - @staticmethod - def sanitize_tx_broadcast_response(server_msg) -> str: - # Unfortunately, bitcoind and hence the Electrum protocol doesn't return a useful error code. - # So, we use substring matching to grok the error message. - # server_msg is untrusted input so it should not be shown to the user. see #4968 - server_msg = str(server_msg) - server_msg = server_msg.replace("\n", r"\n") - - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/script/script_error.cpp - script_error_messages = { - r"Script evaluated without error but finished with a false/empty top stack element", - r"Script failed an OP_VERIFY operation", - r"Script failed an OP_EQUALVERIFY operation", - r"Script failed an OP_CHECKMULTISIGVERIFY operation", - r"Script failed an OP_CHECKSIGVERIFY operation", - r"Script failed an OP_NUMEQUALVERIFY operation", - r"Script is too big", - r"Push value size limit exceeded", - r"Operation limit exceeded", - r"Stack size limit exceeded", - r"Signature count negative or greater than pubkey count", - r"Pubkey count negative or limit exceeded", - r"Opcode missing or not understood", - r"Attempted to use a disabled opcode", - r"Operation not valid with the current stack size", - r"Operation not valid with the current altstack size", - r"OP_RETURN was encountered", - r"Invalid OP_IF construction", - r"Negative locktime", - r"Locktime requirement not satisfied", - r"Signature hash type missing or not understood", - r"Non-canonical DER signature", - r"Data push larger than necessary", - r"Only push operators allowed in signatures", - r"Non-canonical signature: S value is unnecessarily high", - r"Dummy CHECKMULTISIG argument must be zero", - r"OP_IF/NOTIF argument must be minimal", - r"Signature must be zero for failed CHECK(MULTI)SIG operation", - r"NOPx reserved for soft-fork upgrades", - r"Witness version reserved for soft-fork upgrades", - r"Taproot version reserved for soft-fork upgrades", - r"OP_SUCCESSx reserved for soft-fork upgrades", - r"Public key version reserved for soft-fork upgrades", - r"Public key is neither compressed or uncompressed", - r"Stack size must be exactly one after execution", - r"Extra items left on stack after execution", - r"Witness program has incorrect length", - r"Witness program was passed an empty witness", - r"Witness program hash mismatch", - r"Witness requires empty scriptSig", - r"Witness requires only-redeemscript scriptSig", - r"Witness provided for non-witness script", - r"Using non-compressed keys in segwit", - r"Invalid Schnorr signature size", - r"Invalid Schnorr signature hash type", - r"Invalid Schnorr signature", - r"Invalid Taproot control block size", - r"Too much signature validation relative to witness weight", - r"OP_CHECKMULTISIG(VERIFY) is not available in tapscript", - r"OP_IF/NOTIF argument must be minimal in tapscript", - r"Using OP_CODESEPARATOR in non-witness script", - r"Signature is found in scriptCode", - } - for substring in script_error_messages: - if substring in server_msg: - return substring - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/validation.cpp - # grep "REJECT_" - # grep "TxValidationResult" - # should come after script_error.cpp (due to e.g. "non-mandatory-script-verify-flag") - validation_error_messages = { - r"coinbase": None, - r"tx-size-small": None, - r"non-final": None, - r"txn-already-in-mempool": None, - r"txn-mempool-conflict": None, - r"txn-already-known": None, - r"non-BIP68-final": None, - r"bad-txns-nonstandard-inputs": None, - r"bad-witness-nonstandard": None, - r"bad-txns-too-many-sigops": None, - r"mempool min fee not met": - ("mempool min fee not met\n" + - _("Your transaction is paying a fee that is so low that the bitcoin node cannot " - "fit it into its mempool. The mempool is already full of hundreds of megabytes " - "of transactions that all pay higher fees. Try to increase the fee.")), - r"min relay fee not met": None, - r"absurdly-high-fee": None, - r"max-fee-exceeded": None, - r"too-long-mempool-chain": None, - r"bad-txns-spends-conflicting-tx": None, - r"insufficient fee": ("insufficient fee\n" + - _("Your transaction is trying to replace another one in the mempool but it " - "does not meet the rules to do so. Try to increase the fee.")), - r"too many potential replacements": None, - r"replacement-adds-unconfirmed": None, - r"mempool full": None, - r"non-mandatory-script-verify-flag": None, - r"mandatory-script-verify-flag-failed": None, - r"Transaction check failed": None, - } - for substring in validation_error_messages: - if substring in server_msg: - msg = validation_error_messages[substring] - return msg if msg else substring - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/rpc/rawtransaction.cpp - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/util/error.cpp - # https://github.com/bitcoin/bitcoin/blob/3f83c744ac28b700090e15b5dda2260724a56f49/src/common/messages.cpp#L126 - # grep "RPC_TRANSACTION" - # grep "RPC_DESERIALIZATION_ERROR" - # grep "TransactionError" - rawtransaction_error_messages = { - r"Missing inputs": None, - r"Inputs missing or spent": None, - r"transaction already in block chain": None, - r"Transaction already in block chain": None, - r"Transaction outputs already in utxo set": None, - r"TX decode failed": None, - r"Peer-to-peer functionality missing or disabled": None, - r"Transaction rejected by AcceptToMemoryPool": None, - r"AcceptToMemoryPool failed": None, - r"Transaction rejected by mempool": None, - r"Mempool internal error": None, - r"Fee exceeds maximum configured by user": None, - r"Unspendable output exceeds maximum configured by user": None, - r"Transaction rejected due to invalid package": None, - } - for substring in rawtransaction_error_messages: - if substring in server_msg: - msg = rawtransaction_error_messages[substring] - return msg if msg else substring - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/consensus/tx_verify.cpp - # https://github.com/bitcoin/bitcoin/blob/c7ad94428ab6f54661d7a5441e1fdd0ebf034903/src/consensus/tx_check.cpp - # grep "REJECT_" - # grep "TxValidationResult" - tx_verify_error_messages = { - r"bad-txns-vin-empty": None, - r"bad-txns-vout-empty": None, - r"bad-txns-oversize": None, - r"bad-txns-vout-negative": None, - r"bad-txns-vout-toolarge": None, - r"bad-txns-txouttotal-toolarge": None, - r"bad-txns-inputs-duplicate": None, - r"bad-cb-length": None, - r"bad-txns-prevout-null": None, - r"bad-txns-inputs-missingorspent": - ("bad-txns-inputs-missingorspent\n" + - _("You might have a local transaction in your wallet that this transaction " - "builds on top. You need to either broadcast or remove the local tx.")), - r"bad-txns-premature-spend-of-coinbase": None, - r"bad-txns-inputvalues-outofrange": None, - r"bad-txns-in-belowout": None, - r"bad-txns-fee-outofrange": None, - } - for substring in tx_verify_error_messages: - if substring in server_msg: - msg = tx_verify_error_messages[substring] - return msg if msg else substring - # https://github.com/bitcoin/bitcoin/blob/5bb64acd9d3ced6e6f95df282a1a0f8b98522cb0/src/policy/policy.cpp - # grep "reason =" - # should come after validation.cpp (due to "tx-size" vs "tx-size-small") - # should come after script_error.cpp (due to e.g. "version") - policy_error_messages = { - r"version": _("Transaction uses non-standard version."), - r"tx-size": _("The transaction was rejected because it is too large (in bytes)."), - r"scriptsig-size": None, - r"scriptsig-not-pushonly": None, - r"scriptpubkey": - ("scriptpubkey\n" + - _("Some of the outputs pay to a non-standard script.")), - r"bare-multisig": None, - r"dust": - (_("Transaction could not be broadcast due to dust outputs.\n" - "Some of the outputs are too small in value, probably lower than 1000 satoshis.\n" - "Check the units, make sure you haven't confused e.g. mBTC and BTC.")), - r"multi-op-return": _("The transaction was rejected because it contains multiple OP_RETURN outputs."), - } - for substring in policy_error_messages: - if substring in server_msg: - msg = policy_error_messages[substring] - return msg if msg else substring - # otherwise: - return _("Unknown error") - @best_effort_reliable @catch_server_exceptions async def get_transaction(self, tx_hash: str, *, timeout=None) -> str: From 8efc2aab5e5098177c21b6f0ae2e49e8c75efcd3 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 7 Aug 2025 15:52:50 +0000 Subject: [PATCH 2/8] tests: interface: implement toy electrum server --- electrum/interface.py | 5 ++ tests/__init__.py | 10 ++- tests/test_interface.py | 141 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 152 insertions(+), 4 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index c9f438e47..2411a06b0 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -529,6 +529,7 @@ class Interface(Logger): assert isinstance(server, ServerAddr), f"expected ServerAddr, got {type(server)}" self.ready = network.asyncio_loop.create_future() self.got_disconnected = asyncio.Event() + self._blockchain_updated = asyncio.Event() self.server = server Logger.__init__(self) assert network.config.path @@ -1058,6 +1059,8 @@ class Interface(Logger): self.logger.info(f"new chain tip. {height=}") if blockchain_updated: util.trigger_callback('blockchain_updated') + self._blockchain_updated.set() + self._blockchain_updated.clear() util.trigger_callback('network_updated') await self.network.switch_unwanted_fork_interface() await self.network.switch_lagging_interface() @@ -1100,6 +1103,8 @@ class Interface(Logger): continue # report progress to gui/etc util.trigger_callback('blockchain_updated') + self._blockchain_updated.set() + self._blockchain_updated.clear() util.trigger_callback('network_updated') height += num_headers assert height <= next_height+1, (height, self.tip) diff --git a/tests/__init__.py b/tests/__init__.py index 624a3e3e0..51decf3bb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,6 +28,7 @@ class ElectrumTestCase(unittest.IsolatedAsyncioTestCase, Logger): """Base class for our unit tests.""" TESTNET = False + REGTEST = False TEST_ANCHOR_CHANNELS = False # maxDiff = None # for debugging @@ -41,19 +42,22 @@ class ElectrumTestCase(unittest.IsolatedAsyncioTestCase, Logger): @classmethod def setUpClass(cls): super().setUpClass() - if cls.TESTNET: + assert not (cls.REGTEST and cls.TESTNET), "regtest and testnet are mutually exclusive" + if cls.REGTEST: + constants.BitcoinRegtest.set_as_network() + elif cls.TESTNET: constants.BitcoinTestnet.set_as_network() @classmethod def tearDownClass(cls): super().tearDownClass() - if cls.TESTNET: + if cls.TESTNET or cls.REGTEST: constants.BitcoinMainnet.set_as_network() def setUp(self): self._test_lock.acquire() super().setUp() - self.electrum_path = tempfile.mkdtemp() + self.electrum_path = tempfile.mkdtemp(prefix="electrum-unittest-base-") assert util._asyncio_event_loop is None, "global event loop already set?!" async def asyncSetUp(self): diff --git a/tests/test_interface.py b/tests/test_interface.py index 84e223e09..e84026ba0 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -1,4 +1,12 @@ -from electrum.interface import ServerAddr +import asyncio + +import aiorpcx + +from electrum.interface import ServerAddr, Interface, PaddedRSTransport +from electrum import util, blockchain +from electrum.util import OldTaskGroup, bfh +from electrum.logging import Logger +from electrum.simple_config import SimpleConfig from . import ElectrumTestCase @@ -46,3 +54,134 @@ class TestServerAddr(ElectrumTestCase): ServerAddr(host="2400:6180:0:d1::86b:e001", port=50002, protocol="s").to_friendly_name()) self.assertEqual("[2400:6180:0:d1::86b:e001]:50001:t", ServerAddr(host="2400:6180:0:d1::86b:e001", port=50001, protocol="t").to_friendly_name()) + + +class MockNetwork: + + def __init__(self, *, config: SimpleConfig): + self.config = config + self.asyncio_loop = util.get_asyncio_loop() + self.taskgroup = OldTaskGroup() + blockchain.read_blockchains(self.config) + blockchain.init_headers_file_for_best_chain() + self.proxy = None + self.debug = True + self.bhi_lock = asyncio.Lock() + + async def connection_down(self, interface: Interface): + pass + def get_network_timeout_seconds(self, request_type) -> int: + return 10 + def check_interface_against_healthy_spread_of_connected_servers(self, iface_to_check: Interface) -> bool: + return True + def update_fee_estimates(self, *, fee_est: dict[int, int] = None): + pass + async def switch_unwanted_fork_interface(self): + pass + async def switch_lagging_interface(self): + pass + + +# regtest chain: +BLOCK_HEADERS = { + 0: bfh("0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff7f2002000000"), + 1: bfh("0000002006226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f186c8dfd970a4545f79916bc1d75c9d00432f57c89209bf3bb115b7612848f509c25f45bffff7f2000000000"), + 2: bfh("00000020686bdfc6a3db73d5d93e8c9663a720a26ecb1ef20eb05af11b36cdbc57c19f7ebf2cbf153013a1c54abaf70e95198fcef2f3059cc6b4d0f7e876808e7d24d11cc825f45bffff7f2000000000"), + 3: bfh("00000020122baa14f3ef54985ae546d1611559e3f487bd2a0f46e8dbb52fbacc9e237972e71019d7feecd9b8596eca9a67032c5f4641b23b5d731dc393e37de7f9c2f299e725f45bffff7f2000000000"), + 4: bfh("00000020f8016f7ef3a17d557afe05d4ea7ab6bde1b2247b7643896c1b63d43a1598b747a3586da94c71753f27c075f57f44faf913c31177a0957bbda42e7699e3a2141aed25f45bffff7f2001000000"), + 5: bfh("000000201d589c6643c1d121d73b0573e5ee58ab575b8fdf16d507e7e915c5fbfbbfd05e7aee1d692d1615c3bdf52c291032144ce9e3b258a473c17c745047f3431ff8e2ee25f45bffff7f2000000000"), + 6: bfh("00000020b833ed46eea01d4c980f59feee44a66aa1162748b6801029565d1466790c405c3a141ce635cbb1cd2b3a4fcdd0a3380517845ba41736c82a79cab535d31128066526f45bffff7f2001000000"), + 7: bfh("00000020abe8e119d1877c9dc0dc502d1a253fb9a67967c57732d2f71ee0280e8381ff0a9690c2fe7c1a4450c74dc908fe94dd96c3b0637d51475e9e06a78e944a0c7fe28126f45bffff7f2000000000"), + 8: bfh("000000202ce41d94eb70e1518bc1f72523f84a903f9705d967481e324876e1f8cf4d3452148be228a4c3f2061bafe7efdfc4a8d5a94759464b9b5c619994d45dfcaf49e1a126f45bffff7f2000000000"), + 9: bfh("00000020552755b6c59f3d51e361d16281842a4e166007799665b5daed86a063dd89857415681cb2d00ff889193f6a68a93f5096aeb2d84ca0af6185a462555822552221a626f45bffff7f2000000000"), + 10: bfh("00000020a13a491cbefc93cd1bb1938f19957e22a134faf14c7dee951c45533e2c750f239dc087fc977b06c24a69c682d1afd1020e6dc1f087571ccec66310a786e1548fab26f45bffff7f2000000000"), + 11: bfh("00000020dbf3a9b55dfefbaf8b6e43a89cf833fa2e208bbc0c1c5d76c0d71b9e4a65337803b243756c25053253aeda309604363460a3911015929e68705bd89dff6fe064b026f45bffff7f2002000000"), + 12: bfh("000000203d0932b3b0c78eccb39a595a28ae4a7c966388648d7783fd1305ec8d40d4fe5fd67cb902a7d807cee7676cb543feec3e053aa824d5dfb528d5b94f9760313d9db726f45bffff7f2001000000"), +} + +_active_server_sessions = set() +def _get_active_server_session() -> 'ServerSession': + assert 1 == len(_active_server_sessions), len(_active_server_sessions) + return list(_active_server_sessions)[0] + +class ServerSession(aiorpcx.RPCSession, Logger): + + def __init__(self, *args, **kwargs): + aiorpcx.RPCSession.__init__(self, *args, **kwargs) + Logger.__init__(self) + self.logger.debug(f'connection from {self.remote_address()}') + self.cur_height = 6 # type: int # chain tip + _active_server_sessions.add(self) + + async def connection_lost(self): + await super().connection_lost() + self.logger.debug(f'{self.remote_address()} disconnected') + _active_server_sessions.discard(self) + + async def handle_request(self, request): + handlers = { + 'server.version': self._handle_server_version, + 'blockchain.estimatefee': self._handle_estimatefee, + 'blockchain.headers.subscribe': self._handle_headers_subscribe, + 'blockchain.block.header': self._handle_block_header, + 'blockchain.block.headers': self._handle_block_headers, + 'server.ping': self._handle_ping, + } + handler = handlers.get(request.method) + coro = aiorpcx.handler_invocation(handler, request)() + return await coro + + async def _handle_server_version(self, client_name='', protocol_version=None): + return ['best_server_impl/0.1', '1.4'] + + async def _handle_estimatefee(self, number, mode=None): + return 1000 + + async def _handle_headers_subscribe(self): + return {'hex': BLOCK_HEADERS[self.cur_height].hex(), 'height': self.cur_height} + + async def _handle_block_header(self, height): + return BLOCK_HEADERS[height].hex() + + async def _handle_block_headers(self, start_height, count): + assert start_height <= self.cur_height, (start_height, self.cur_height) + last_height = min(start_height+count-1, self.cur_height) # [start_height, last_height] + count = last_height - start_height + 1 + headers = b"".join(BLOCK_HEADERS[idx] for idx in range(start_height, last_height+1)) + return {'hex': headers.hex(), 'count': count, 'max': 2016} + + async def _handle_ping(self): + return None + + +class TestInterface(ElectrumTestCase): + REGTEST = True + + def setUp(self): + super().setUp() + self.config = SimpleConfig({'electrum_path': self.electrum_path}) + self._orig_WAIT_FOR_BUFFER_GROWTH_SECONDS = PaddedRSTransport.WAIT_FOR_BUFFER_GROWTH_SECONDS + PaddedRSTransport.WAIT_FOR_BUFFER_GROWTH_SECONDS = 0 + + def tearDown(self): + PaddedRSTransport.WAIT_FOR_BUFFER_GROWTH_SECONDS = self._orig_WAIT_FOR_BUFFER_GROWTH_SECONDS + super().tearDown() + + async def asyncSetUp(self): + await super().asyncSetUp() + self._server: asyncio.base_events.Server = await aiorpcx.serve_rs(ServerSession, "127.0.0.1") + server_socket_addr = self._server.sockets[0].getsockname() + self._server_port = server_socket_addr[1] + self.network = MockNetwork(config=self.config) + + async def asyncTearDown(self): + self._server.close() + await super().asyncTearDown() + + async def test_client_syncs_headers_to_tip(self): + interface = Interface(network=self.network, server=ServerAddr(host="127.0.0.1", port=self._server_port, protocol="t")) + self.network.interface = interface + await util.wait_for2(interface.ready, 5) + await interface._blockchain_updated.wait() + self.assertEqual(_get_active_server_session().cur_height, interface.tip) + self.assertFalse(interface.got_disconnected.is_set()) From d47f38b0e0762938576305765612952b687f1f4a Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 7 Aug 2025 16:16:50 +0000 Subject: [PATCH 3/8] tests: interface: impl get_tx/broadcast_tx for electrum server --- tests/test_interface.py | 45 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/test_interface.py b/tests/test_interface.py index e84026ba0..47310853d 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -1,12 +1,15 @@ import asyncio import aiorpcx +from aiorpcx import RPCError +import electrum from electrum.interface import ServerAddr, Interface, PaddedRSTransport from electrum import util, blockchain from electrum.util import OldTaskGroup, bfh from electrum.logging import Logger from electrum.simple_config import SimpleConfig +from electrum.transaction import Transaction from . import ElectrumTestCase @@ -111,6 +114,9 @@ class ServerSession(aiorpcx.RPCSession, Logger): Logger.__init__(self) self.logger.debug(f'connection from {self.remote_address()}') self.cur_height = 6 # type: int # chain tip + self.txs = { + "bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb": bfh("020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025100ffffffff0200f2052a010000001600140297bde2689a3c79ffe050583b62f86f2d9dae540000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000"), + } # type: dict[str, bytes] _active_server_sessions.add(self) async def connection_lost(self): @@ -125,6 +131,8 @@ class ServerSession(aiorpcx.RPCSession, Logger): 'blockchain.headers.subscribe': self._handle_headers_subscribe, 'blockchain.block.header': self._handle_block_header, 'blockchain.block.headers': self._handle_block_headers, + 'blockchain.transaction.get': self._handle_transaction_get, + 'blockchain.transaction.broadcast': self._handle_transaction_broadcast, 'server.ping': self._handle_ping, } handler = handlers.get(request.method) @@ -153,6 +161,19 @@ class ServerSession(aiorpcx.RPCSession, Logger): async def _handle_ping(self): return None + async def _handle_transaction_get(self, tx_hash: str, verbose=False): + assert not verbose + rawtx = self.txs.get(tx_hash) + if rawtx is None: + DAEMON_ERROR = 2 + raise RPCError(DAEMON_ERROR, f'daemon error: unknown txid={tx_hash}') + return rawtx.hex() + + async def _handle_transaction_broadcast(self, raw_tx: str): + tx = Transaction(raw_tx) + self.txs[tx.txid()] = bfh(raw_tx) + return tx.txid() + class TestInterface(ElectrumTestCase): REGTEST = True @@ -178,10 +199,32 @@ class TestInterface(ElectrumTestCase): self._server.close() await super().asyncTearDown() - async def test_client_syncs_headers_to_tip(self): + async def _start_iface_and_wait_for_sync(self): interface = Interface(network=self.network, server=ServerAddr(host="127.0.0.1", port=self._server_port, protocol="t")) self.network.interface = interface await util.wait_for2(interface.ready, 5) await interface._blockchain_updated.wait() + return interface + + async def test_client_syncs_headers_to_tip(self): + interface = await self._start_iface_and_wait_for_sync() self.assertEqual(_get_active_server_session().cur_height, interface.tip) self.assertFalse(interface.got_disconnected.is_set()) + + async def test_transaction_get(self): + interface = await self._start_iface_and_wait_for_sync() + # try requesting tx unknown to server: + with self.assertRaises(RPCError) as ctx: + await interface.get_transaction("deadbeef"*8) + self.assertTrue("unknown txid" in ctx.exception.message) + # try requesting known tx: + rawtx = await interface.get_transaction("bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb") + self.assertEqual(rawtx, _get_active_server_session().txs["bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb"].hex()) + + async def test_transaction_broadcast(self): + interface = await self._start_iface_and_wait_for_sync() + rawtx1 = "020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025200ffffffff0200f2052a010000001600140297bde2689a3c79ffe050583b62f86f2d9dae540000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000" + tx = Transaction(rawtx1) + await interface.broadcast_transaction(tx) + rawtx2 = await interface.get_transaction(tx.txid()) + self.assertEqual(rawtx1, rawtx2) From 138e2f6ba05245394ef89b4fedc3cc4ea3057e55 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 6 Aug 2025 13:14:20 +0000 Subject: [PATCH 4/8] add "lrucache" module, extracted from the 3rd-party tkem/cachetools library functools.lru_cache in the stdlib is not generic enough. That can be used to cache the results of a single pure function, however I have usecases where one function is supposed to populate the cache, while another function consumes it. https://docs.python.org/3/library/functools.html#functools.lru_cache This is stripped down and extracts just the LRUCache from tkem/cachetools. It is relatively short, and very mature code. I don't expect that we have to "follow" upstream, etc. There likely won't be relevant changes upstream. Effectively, we are forking and bundling this code. similar to https://github.com/spesmilo/electrumx/commit/04582cc353b45721e2461176cfce1768003dd5ae --- electrum/lrucache.py | 181 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 electrum/lrucache.py diff --git a/electrum/lrucache.py b/electrum/lrucache.py new file mode 100644 index 000000000..f6c09565a --- /dev/null +++ b/electrum/lrucache.py @@ -0,0 +1,181 @@ +# The MIT License (MIT) +# +# Copyright (c) 2014-2022 Thomas Kemmer +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# ----- +# +# This is a stripped down LRU-cache from the "cachetools" library. +# https://github.com/tkem/cachetools/blob/d991ac71b4eb6394be5ec572b835434081393215/src/cachetools/__init__.py + +import collections +import collections.abc + + +class _DefaultSize: + + __slots__ = () + + def __getitem__(self, _): + return 1 + + def __setitem__(self, _, value): + assert value == 1 + + def pop(self, _): + return 1 + + +class Cache(collections.abc.MutableMapping): + """Mutable mapping to serve as a simple cache or cache base class.""" + + __marker = object() + + __size = _DefaultSize() + + def __init__(self, maxsize, getsizeof=None): + if getsizeof: + self.getsizeof = getsizeof + if self.getsizeof is not Cache.getsizeof: + self.__size = dict() + self.__data = dict() + self.__currsize = 0 + self.__maxsize = maxsize + + def __repr__(self): + return "%s(%s, maxsize=%r, currsize=%r)" % ( + self.__class__.__name__, + repr(self.__data), + self.__maxsize, + self.__currsize, + ) + + def __getitem__(self, key): + try: + return self.__data[key] + except KeyError: + return self.__missing__(key) + + def __setitem__(self, key, value): + maxsize = self.__maxsize + size = self.getsizeof(value) + if size > maxsize: + raise ValueError("value too large") + if key not in self.__data or self.__size[key] < size: + while self.__currsize + size > maxsize: + self.popitem() + if key in self.__data: + diffsize = size - self.__size[key] + else: + diffsize = size + self.__data[key] = value + self.__size[key] = size + self.__currsize += diffsize + + def __delitem__(self, key): + size = self.__size.pop(key) + del self.__data[key] + self.__currsize -= size + + def __contains__(self, key): + return key in self.__data + + def __missing__(self, key): + raise KeyError(key) + + def __iter__(self): + return iter(self.__data) + + def __len__(self): + return len(self.__data) + + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + + def pop(self, key, default=__marker): + if key in self: + value = self[key] + del self[key] + elif default is self.__marker: + raise KeyError(key) + else: + value = default + return value + + def setdefault(self, key, default=None): + if key in self: + value = self[key] + else: + self[key] = value = default + return value + + @property + def maxsize(self): + """The maximum size of the cache.""" + return self.__maxsize + + @property + def currsize(self): + """The current size of the cache.""" + return self.__currsize + + @staticmethod + def getsizeof(value): + """Return the size of a cache element's value.""" + return 1 + + +class LRUCache(Cache): + """Least Recently Used (LRU) cache implementation.""" + + def __init__(self, maxsize, getsizeof=None): + Cache.__init__(self, maxsize, getsizeof) + self.__order = collections.OrderedDict() + + def __getitem__(self, key, cache_getitem=Cache.__getitem__): + value = cache_getitem(self, key) + if key in self: # __missing__ may not store item + self.__update(key) + return value + + def __setitem__(self, key, value, cache_setitem=Cache.__setitem__): + cache_setitem(self, key, value) + self.__update(key) + + def __delitem__(self, key, cache_delitem=Cache.__delitem__): + cache_delitem(self, key) + del self.__order[key] + + def popitem(self): + """Remove and return the `(key, value)` pair least recently used.""" + try: + key = next(iter(self.__order)) + except StopIteration: + raise KeyError("%s is empty" % type(self).__name__) from None + else: + return (key, self.pop(key)) + + def __update(self, key): + try: + self.__order.move_to_end(key) + except KeyError: + self.__order[key] = None From 427b0d42b61c14729bb6b8f725e190d3f0a0f9b4 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Thu, 7 Aug 2025 16:53:32 +0000 Subject: [PATCH 5/8] lrucache: add type hints --- electrum/lrucache.py | 45 +++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/electrum/lrucache.py b/electrum/lrucache.py index f6c09565a..9f81984d7 100644 --- a/electrum/lrucache.py +++ b/electrum/lrucache.py @@ -26,6 +26,7 @@ import collections import collections.abc +from typing import TypeVar, Dict class _DefaultSize: @@ -42,19 +43,21 @@ class _DefaultSize: return 1 -class Cache(collections.abc.MutableMapping): +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +class Cache(collections.abc.MutableMapping[_KT, _VT]): """Mutable mapping to serve as a simple cache or cache base class.""" __marker = object() __size = _DefaultSize() - def __init__(self, maxsize, getsizeof=None): + def __init__(self, maxsize: int, getsizeof=None): if getsizeof: self.getsizeof = getsizeof if self.getsizeof is not Cache.getsizeof: self.__size = dict() - self.__data = dict() + self.__data = dict() # type: Dict[_KT, _VT] self.__currsize = 0 self.__maxsize = maxsize @@ -66,13 +69,13 @@ class Cache(collections.abc.MutableMapping): self.__currsize, ) - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> _VT: try: return self.__data[key] except KeyError: return self.__missing__(key) - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: maxsize = self.__maxsize size = self.getsizeof(value) if size > maxsize: @@ -88,15 +91,15 @@ class Cache(collections.abc.MutableMapping): self.__size[key] = size self.__currsize += diffsize - def __delitem__(self, key): + def __delitem__(self, key: _KT) -> None: size = self.__size.pop(key) del self.__data[key] self.__currsize -= size - def __contains__(self, key): + def __contains__(self, key: _KT) -> bool: return key in self.__data - def __missing__(self, key): + def __missing__(self, key: _KT): raise KeyError(key) def __iter__(self): @@ -105,13 +108,13 @@ class Cache(collections.abc.MutableMapping): def __len__(self): return len(self.__data) - def get(self, key, default=None): + def get(self, key: _KT, default: _VT = None) -> _VT | None: if key in self: return self[key] else: return default - def pop(self, key, default=__marker): + def pop(self, key: _KT, default=__marker) -> _VT: if key in self: value = self[key] del self[key] @@ -121,7 +124,7 @@ class Cache(collections.abc.MutableMapping): value = default return value - def setdefault(self, key, default=None): + def setdefault(self, key: _KT, default: _VT = None) -> _VT | None: if key in self: value = self[key] else: @@ -129,43 +132,43 @@ class Cache(collections.abc.MutableMapping): return value @property - def maxsize(self): + def maxsize(self) -> int: """The maximum size of the cache.""" return self.__maxsize @property - def currsize(self): + def currsize(self) -> int: """The current size of the cache.""" return self.__currsize @staticmethod - def getsizeof(value): + def getsizeof(value) -> int: """Return the size of a cache element's value.""" return 1 -class LRUCache(Cache): +class LRUCache(Cache[_KT, _VT]): """Least Recently Used (LRU) cache implementation.""" - def __init__(self, maxsize, getsizeof=None): + def __init__(self, maxsize: int, getsizeof=None): Cache.__init__(self, maxsize, getsizeof) self.__order = collections.OrderedDict() - def __getitem__(self, key, cache_getitem=Cache.__getitem__): + def __getitem__(self, key: _KT, cache_getitem=Cache.__getitem__) -> _VT | None: value = cache_getitem(self, key) if key in self: # __missing__ may not store item self.__update(key) return value - def __setitem__(self, key, value, cache_setitem=Cache.__setitem__): + def __setitem__(self, key: _KT, value, cache_setitem=Cache.__setitem__) -> None: cache_setitem(self, key, value) self.__update(key) - def __delitem__(self, key, cache_delitem=Cache.__delitem__): + def __delitem__(self, key: _KT, cache_delitem=Cache.__delitem__) -> None: cache_delitem(self, key) del self.__order[key] - def popitem(self): + def popitem(self) -> tuple[_KT, _VT]: """Remove and return the `(key, value)` pair least recently used.""" try: key = next(iter(self.__order)) @@ -174,7 +177,7 @@ class LRUCache(Cache): else: return (key, self.pop(key)) - def __update(self, key): + def __update(self, key: _KT) -> None: try: self.__order.move_to_end(key) except KeyError: From 05da50178b24ed9ffef657d0c28f30e16e0fe0ad Mon Sep 17 00:00:00 2001 From: SomberNight Date: Wed, 6 Aug 2025 13:49:43 +0000 Subject: [PATCH 6/8] interface: don't request same tx from server that we just broadcast to it Often when the wallet creates a tx, the flow is: - create unsigned tx - sign tx - broadcast tx, but don't save it in history - server sends notification that status of a subscribed address changed - client calls scripthash.get_history - client sees txid in scripthash.get_history response - client calls blockchain.transaction.get to request missing tx Instead, now when we broadcast a tx on an interface, we cache that tx *for that interface*, and just before calling blockchain.transaction.get, we lookup in the cache. Hence this will often save a network request. --- electrum/interface.py | 19 ++++++++++++++++--- tests/test_interface.py | 10 ++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/electrum/interface.py b/electrum/interface.py index 2411a06b0..36cfb13c4 100644 --- a/electrum/interface.py +++ b/electrum/interface.py @@ -64,6 +64,7 @@ from .i18n import _ from .logging import Logger from .transaction import Transaction from .fee_policy import FEE_ETA_TARGETS +from .lrucache import LRUCache if TYPE_CHECKING: from .network import Network @@ -558,6 +559,7 @@ class Interface(Logger): self.tip = 0 self._headers_cache = {} # type: Dict[int, bytes] + self._rawtx_cache = LRUCache(maxsize=20) # type: LRUCache[str, bytes] # txid->rawtx self.fee_estimates_eta = {} # type: Dict[int, int] @@ -1318,6 +1320,8 @@ class Interface(Logger): async def get_transaction(self, tx_hash: str, *, timeout=None) -> str: if not is_hash256_str(tx_hash): raise Exception(f"{repr(tx_hash)} is not a txid") + if rawtx_bytes := self._rawtx_cache.get(tx_hash): + return rawtx_bytes.hex() raw = await self.session.send_request('blockchain.transaction.get', [tx_hash], timeout=timeout) # validate response if not is_hex_str(raw): @@ -1329,16 +1333,21 @@ class Interface(Logger): raise RequestCorrupted(f"cannot deserialize received transaction (txid {tx_hash})") from e if tx.txid() != tx_hash: raise RequestCorrupted(f"received tx does not match expected txid {tx_hash} (got {tx.txid()})") + self._rawtx_cache[tx_hash] = bytes.fromhex(raw) return raw async def broadcast_transaction(self, tx: 'Transaction', *, timeout=None) -> None: """caller should handle TxBroadcastError and RequestTimedOut""" + txid_calc = tx.txid() + assert txid_calc is not None + rawtx = tx.serialize() + assert is_hex_str(rawtx) if timeout is None: timeout = self.network.get_network_timeout_seconds(NetworkTimeout.Urgent) if any(DummyAddress.is_dummy_address(txout.address) for txout in tx.outputs()): raise DummyAddressUsedInTxException("tried to broadcast tx with dummy address!") try: - out = await self.session.send_request('blockchain.transaction.broadcast', [tx.serialize()], timeout=timeout) + out = await self.session.send_request('blockchain.transaction.broadcast', [rawtx], timeout=timeout) # note: both 'out' and exception messages are untrusted input from the server except (RequestTimedOut, asyncio.CancelledError, asyncio.TimeoutError): raise # pass-through @@ -1349,10 +1358,14 @@ class Interface(Logger): self.logger.info(f"broadcast_transaction error2 [DO NOT TRUST THIS MESSAGE]: {error_text_str_to_safe_str(repr(e))}. tx={str(tx)}") send_exception_to_crash_reporter(e) raise TxBroadcastUnknownError() from e - if out != tx.txid(): + if out != txid_calc: self.logger.info(f"unexpected txid for broadcast_transaction [DO NOT TRUST THIS MESSAGE]: " - f"{error_text_str_to_safe_str(out)} != {tx.txid()}. tx={str(tx)}") + f"{error_text_str_to_safe_str(out)} != {txid_calc}. tx={str(tx)}") raise TxBroadcastHashMismatch(_("Server returned unexpected transaction ID.")) + # broadcast succeeded. + # We now cache the rawtx, for *this interface only*. The tx likely touches some ismine addresses, affecting + # the status of a scripthash we are subscribed to. Caching here will save a future get_transaction RPC. + self._rawtx_cache[txid_calc] = bytes.fromhex(rawtx) async def get_history_for_scripthash(self, sh: str) -> List[dict]: if not is_hash256_str(sh): diff --git a/tests/test_interface.py b/tests/test_interface.py index 47310853d..aa54f9bdd 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -1,4 +1,5 @@ import asyncio +import collections import aiorpcx from aiorpcx import RPCError @@ -117,6 +118,7 @@ class ServerSession(aiorpcx.RPCSession, Logger): self.txs = { "bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb": bfh("020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025100ffffffff0200f2052a010000001600140297bde2689a3c79ffe050583b62f86f2d9dae540000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000"), } # type: dict[str, bytes] + self._method_counts = collections.defaultdict(int) # type: dict[str, int] _active_server_sessions.add(self) async def connection_lost(self): @@ -136,6 +138,7 @@ class ServerSession(aiorpcx.RPCSession, Logger): 'server.ping': self._handle_ping, } handler = handlers.get(request.method) + self._method_counts[request.method] += 1 coro = aiorpcx.handler_invocation(handler, request)() return await coro @@ -220,11 +223,18 @@ class TestInterface(ElectrumTestCase): # try requesting known tx: rawtx = await interface.get_transaction("bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb") self.assertEqual(rawtx, _get_active_server_session().txs["bdae818ad3c1f261317738ae9284159bf54874356f186dbc7afd631dc1527fcb"].hex()) + self.assertEqual(_get_active_server_session()._method_counts["blockchain.transaction.get"], 2) async def test_transaction_broadcast(self): interface = await self._start_iface_and_wait_for_sync() rawtx1 = "020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025200ffffffff0200f2052a010000001600140297bde2689a3c79ffe050583b62f86f2d9dae540000000000000000266a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf90120000000000000000000000000000000000000000000000000000000000000000000000000" tx = Transaction(rawtx1) + # broadcast await interface.broadcast_transaction(tx) + self.assertEqual(bfh(rawtx1), _get_active_server_session().txs.get(tx.txid())) + # now request tx. + # as we just broadcast this same tx, this will hit the client iface cache, and won't call the server. + self.assertEqual(_get_active_server_session()._method_counts["blockchain.transaction.get"], 0) rawtx2 = await interface.get_transaction(tx.txid()) self.assertEqual(rawtx1, rawtx2) + self.assertEqual(_get_active_server_session()._method_counts["blockchain.transaction.get"], 0) From cd8bbcd2bb229cdf5c759de8e21f17dcefcc6355 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 8 Aug 2025 14:05:33 +0000 Subject: [PATCH 7/8] tests: don't block forever if a prior unit test raised during setUp --- tests/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 51decf3bb..492c295e1 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -55,7 +55,11 @@ class ElectrumTestCase(unittest.IsolatedAsyncioTestCase, Logger): constants.BitcoinMainnet.set_as_network() def setUp(self): - self._test_lock.acquire() + have_lock = self._test_lock.acquire(timeout=0.1) + if not have_lock: + # This can happen when trying to run the tests in parallel, + # or if a prior test raised during `setUp` or `asyncSetUp` and never released the lock. + raise Exception("timed out waiting for test_lock") super().setUp() self.electrum_path = tempfile.mkdtemp(prefix="electrum-unittest-base-") assert util._asyncio_event_loop is None, "global event loop already set?!" From 98c4c9709f3535f80d7ecee773d1bed5f19994ea Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 8 Aug 2025 14:11:22 +0000 Subject: [PATCH 8/8] tests: interface: more aggressive clean-up --- tests/test_interface.py | 4 ++++ tests/test_lntransport.py | 1 + 2 files changed, 5 insertions(+) diff --git a/tests/test_interface.py b/tests/test_interface.py index aa54f9bdd..da4f607cd 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -71,6 +71,7 @@ class MockNetwork: self.proxy = None self.debug = True self.bhi_lock = asyncio.Lock() + self.interface = None # type: Interface | None async def connection_down(self, interface: Interface): pass @@ -199,7 +200,10 @@ class TestInterface(ElectrumTestCase): self.network = MockNetwork(config=self.config) async def asyncTearDown(self): + if self.network.interface: + await self.network.interface.close() self._server.close() + await self._server.wait_closed() await super().asyncTearDown() async def _start_iface_and_wait_for_sync(self): diff --git a/tests/test_lntransport.py b/tests/test_lntransport.py index 07bd1fc27..bcad665db 100644 --- a/tests/test_lntransport.py +++ b/tests/test_lntransport.py @@ -102,6 +102,7 @@ class TestLNTransport(ElectrumTestCase): for t in transports: t.close() server.close() + await server.wait_closed() await f()