diff --git a/electrum/lnworker.py b/electrum/lnworker.py
index d5a8fc05b..a2b8feb7f 100644
--- a/electrum/lnworker.py
+++ b/electrum/lnworker.py
@@ -2318,6 +2318,13 @@ class LNWallet(LNWorker):
                 return key_list
         return []
 
+    def delete_payment_bundle(self, payment_hash: bytes) -> None:
+        payment_key = self._get_payment_key(payment_hash)
+        for key_list in self.payment_bundles:
+            if payment_key in key_list:
+                self.payment_bundles.remove(key_list)
+                return
+
     def save_preimage(self, payment_hash: bytes, preimage: bytes, *, write_to_disk: bool = True):
         if sha256(preimage) != payment_hash:
             raise Exception("tried to save incorrect preimage for payment_hash")
diff --git a/electrum/submarine_swaps.py b/electrum/submarine_swaps.py
index 9738c9f57..de0041991 100644
--- a/electrum/submarine_swaps.py
+++ b/electrum/submarine_swaps.py
@@ -42,7 +42,7 @@ from .json_db import StoredObject, stored_in
 from . import constants
 from .address_synchronizer import TX_HEIGHT_LOCAL, TX_HEIGHT_FUTURE
 from .fee_policy import FeePolicy
-from .invoices import Invoice
+from .invoices import Invoice, PR_PAID
 from .lnonion import OnionRoutingFailure, OnionFailureCode
 from .lnsweep import SweepInfo
 
@@ -370,16 +370,22 @@ class SwapManager(Logger):
         self.logger.info(f'failing swap {swap.payment_hash.hex()}: {reason}')
         if not swap.is_reverse and swap.payment_hash in self.lnworker.hold_invoice_callbacks:
             self.lnworker.unregister_hold_invoice(swap.payment_hash)
-            payment_secret = self.lnworker.get_payment_secret(swap.payment_hash)
-            payment_key = swap.payment_hash + payment_secret
-            e = OnionRoutingFailure(code=OnionFailureCode.INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS, data=b'')
-            self.lnworker.save_forwarding_failure(payment_key.hex(), failure_message=e)
+            # Peer.maybe_fulfill_htlc will fail incoming htlcs if there is no payment info
+            self.lnworker.delete_payment_info(swap.payment_hash.hex())
+            self.lnworker.clear_invoices_cache()
         self.lnwatcher.remove_callback(swap.lockup_address)
         if not swap.is_funded():
             with self.swaps_lock:
                 if self._swaps.pop(swap.payment_hash.hex(), None) is None:
                     self.logger.debug(f"swap {swap.payment_hash.hex()} has already been deleted.")
-                # TODO clean-up other swaps dicts, i.e. undo _add_or_reindex_swap()
+                if swap._funding_prevout is not None:
+                    self._swaps_by_funding_outpoint.pop(swap._funding_prevout, None)
+                self._swaps_by_lockup_address.pop(swap.lockup_address, None)
+                if swap.prepay_hash is not None:
+                    self._prepayments.pop(swap.prepay_hash, None)
+                    if self.lnworker.get_payment_status(swap.prepay_hash) != PR_PAID:
+                        self.lnworker.delete_payment_info(swap.prepay_hash.hex())
+                        self.lnworker.delete_payment_bundle(swap.payment_hash)
 
     @classmethod
     def extract_preimage(cls, swap: SwapData, claim_tx: Transaction) -> Optional[bytes]:
@@ -433,8 +439,12 @@ class SwapManager(Logger):
                 if spent_height > 0:
                     if current_height - spent_height > REDEEM_AFTER_DOUBLE_SPENT_DELAY:
                         self.logger.info(f'stop watching swap {swap.lockup_address}')
-                        self.lnwatcher.remove_callback(swap.lockup_address)
                         swap.is_redeemed = True
+                        # cleanup
+                        self.lnwatcher.remove_callback(swap.lockup_address)
+                        if not swap.is_reverse:
+                            self.lnworker.delete_payment_bundle(swap.payment_hash)
+                            self.lnworker.unregister_hold_invoice(swap.payment_hash)
 
             if not swap.is_reverse:
                 if swap.preimage is None and spent_height is not None: