container: hledger-flow: electrum: add lightning, backwards compat

*WARNING*: lightning support is a WIP due to an unresolved electrum lightning history bug (see docker-finance #227). This commit is needed for backwards compatibility, as the new electrum format will break import - even if lightning is never used.
2025-10-02 15:35:40 -07:00
parent ccaab13534
commit d684f370ff
2 changed files with 331 additions and 40 deletions
--- a/container/src/hledger-flow/accounts/electrum/electrum-shared.bash
+++ b/container/src/hledger-flow/accounts/electrum/electrum-shared.bash
@@ -34,7 +34,8 @@ source "${DOCKER_FINANCE_CONTAINER_REPO}/src/hledger-flow/lib/lib_preprocess.bas
 [ -z "$global_in_path" ] && exit 1
 [ -z "$global_out_path" ] && exit 1

-function parse()
+# Since electrum 3.3.0
+function parse_without-lightning-header()
 {
  lib_preprocess::assert_header "transaction_hash,label,confirmations,value,fiat_value,fee,fiat_fee,timestamp"

@@ -43,19 +44,49 @@ function parse()
       if (NR<2 || $8 !~ global_year)
         next

-       printf $1 OFS                   # transaction_hash
+       #
+       # transaction_hash (oc_transaction_hash)
+       #
+
+       printf $1 OFS
+
+       #
+       # ln_payment_hash (N/A)
+       #
+
+       printf OFS
+
+       #
+       # label
+       #

       sub(/%/, "%%", $2)
-       printf $2 OFS                   # label
+       printf $2 OFS

-       printf $3 OFS                   # confirmations
+       #
+       # confirmations
+       #

-       # value (without fee)
-       direction=($4 ~ /^-/ ? "OUT" : "IN"); sub(/^-/, "", $4)
-       value=$4-$6  # Remove fee
+       printf $3 OFS
+
+       #
+       # value (amount_chain_bc)
+       #
+
+       direction = ($4 ~ /^-/ ? "OUT" : "IN"); sub(/^-/, "", $4)
+       value = $4 - $6  # Remove included network fee
       printf("%.8f", value); printf OFS

+       #
+       # amount_lightning_bc (N/A)
+       #
+
+       printf OFS
+
+       #
       # fiat_value
+       #
+
       if ($5 !~ /^No Data$/)
         {
           sub(/^-/, "", $5)
@@ -63,22 +94,161 @@ function parse()
         }
       printf OFS

-       # fee
+       #
+       # fee (network_fee_satoshi)
+       #
+
       printf("%.8f", $6)
       printf OFS

+       #
       # fiat_fee
+       #
+
       if ($7 !~ /^No Data$/)
         {
           printf("%.8f", $7)
         }
       printf OFS

+       #
       # timestamp
+       #
+
       sub(/ /, "T", $8)  # HACK: makes arg-friendly by removing space
       cmd = "date \"+%F %T %z\" --date="$8 | getline date
       printf date OFS

+       #
+       # Direction, subaccount
+       #
+
+       printf direction OFS
+       printf global_subaccount
+
+       printf "\n"
+
+    }' FS=, OFS=, "$global_in_path" >"$global_out_path"
+}
+
+# Since electrum 4.6.2
+# TODO: *WARNING*: lightning support is a WIP due to an unresolved electrum lightning history bug (see docker-finance #227)
+function parse_with-lightning-header()
+{
+  lib_preprocess::assert_header "oc_transaction_hash,ln_payment_hash,label,confirmations,amount_chain_bc,amount_lightning_bc,fiat_value,network_fee_satoshi,fiat_fee,timestamp"
+
+  gawk -v global_year="$global_year" -v global_subaccount="$global_subaccount" \
+    '{
+       if (NR<2 || $10 !~ global_year)
+         next
+
+       #
+       # oc_transaction_hash
+       #
+
+       printf $1 OFS
+
+       #
+       # ln_payment_hash
+       #
+
+       printf $2 OFS
+
+       #
+       # label
+       #
+
+       sub(/%/, "%%", $3)
+       printf $3 OFS
+
+       #
+       # confirmations
+       #
+
+       printf $4 OFS
+
+       #
+       # amount_chain_bc
+       #
+
+       direction = ($5 ~ /^-/ ? "OUT" : "IN"); sub(/^-/, "", $5)
+
+       # Network fee is now in satoshis
+       network_fee = $8 * 0.00000001
+
+       # Remove included network fee
+       oc_amount = ($5 > 0 ? $5 - network_fee : $5)
+
+       printf("%.8f", oc_amount); printf OFS
+
+       #
+       # amount_lightning_bc
+       #
+
+       # NOTE: opening a channel will be positive but it is not an IN
+       if (direction == "IN")
+         {
+           if ($6 ~ /^-/)
+             {
+               direction = "OUT"
+             }
+           else
+             {
+               direction = "IN"
+             }
+         }
+       sub(/^-/, "", $6)
+
+       # If lightning hash exists, is not opening/closing a channel
+       if ($2 ~ /[a-z0-9]/)
+         {
+           ln_amount = ($6 > 0 ? $6 - network_fee : $6)
+         }
+       else
+         {
+           ln_amount = $6
+         }
+       printf("%.8f", ln_amount); printf OFS
+
+       #
+       # fiat_value
+       #
+
+       if ($7 !~ /^No Data$|^$/)
+         {
+           sub(/^-/, "", $7)
+           printf("%.8f", $7)
+         }
+       printf OFS
+
+       #
+       # network_fee_satoshi
+       #
+
+       printf("%.8f", network_fee); printf OFS
+
+       #
+       # fiat_fee
+       #
+
+       if ($9 !~ /^No Data$|^$/)
+         {
+           printf("%.8f", $9)
+         }
+       printf OFS
+
+       #
+       # timestamp
+       #
+
+       sub(/ /, "T", $10)  # HACK: makes arg-friendly by removing space
+       cmd = "date \"+%F %T %z\" --date="$10 | getline date
+       printf date OFS
+
+       #
+       # Direction, subaccount
+       #
+
       printf direction OFS
       printf global_subaccount

@@ -89,7 +259,11 @@ function parse()

 function main()
 {
-  parse
+  lib_preprocess::test_header "transaction_hash,label,confirmations,value,fiat_value,fee,fiat_fee,timestamp" \
+    && parse_without-lightning-header && return 0
+
+  lib_preprocess::test_header "oc_transaction_hash,ln_payment_hash,label,confirmations,amount_chain_bc,amount_lightning_bc,fiat_value,network_fee_satoshi,fiat_fee,timestamp" \
+    && parse_with-lightning-header
 }

 main "$@"
--- a/container/src/hledger-flow/accounts/electrum/electrum-shared.rules
+++ b/container/src/hledger-flow/accounts/electrum/electrum-shared.rules
@@ -15,74 +15,191 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.

-fields txid,label,confirmations,value_without_fee,fiat_value,fee,fiat_fee,timestamp,direction,subaccount
+# TODO: *WARNING*: lightning support is a WIP due to an unresolved electrum lightning history bug (see docker-finance #227)
+
+fields oc_hash,ln_hash,label,confirmations,oc_amount,ln_amount,fiat_value,network_fee,fiat_fee,timestamp,direction,subaccount

 # Electrum exports to localtime
 date-format %Y-%m-%d %H:%M:%S %z
 date %timestamp

+# Actual description must be kept as a notes tag
 description %timestamp

-# Treat the Electrum 'label' field as notes
-comment txid:%txid, notes:%label, direction:%direction
-
 account1 assets:electrum:%subaccount:BTC
-amount %value_without_fee BTC
+amount %oc_amount BTC

-if %direction ^OUT$
-  amount -%value_without_fee BTC
+comment txid:%oc_hash, notes:%label, direction:%direction
+
+if %ln_hash [0-9a-z]
+  comment txid:%oc_hash, ln_hash:%ln_hash, notes:%label, direction:%direction

 #
-# Set account based 'label' keywords
-#
-
 # IN
+#

 if %direction ^IN$
  account2 equity:electrum:%subaccount:withdrawal:BTC

+if %direction ^IN$
+& %oc_amount [1-9]
+& %ln_amount [1-9]
+  amount2 -%oc_amount BTC
+  account3 assets:electrum:%subaccount:ln_locked_payable:BTC
+  amount3 %ln_amount BTC
+  account3 assets:electrum:%subaccount:BTC
+  amount4 -%ln_amount BTC
+
+if %direction ^IN$
+& %oc_amount ^[^1-9]*$
+& %ln_amount [1-9]
+  amount %ln_amount BTC
+  amount2 -%ln_amount BTC
+
+# Label: lightning channels
+
+if %direction ^IN$
+& %oc_amount [1-9]
+& %ln_amount [1-9]
+& %label ^Close channel
+  amount %ln_amount BTC
+  account2 assets:electrum:%subaccount:ln_locked_payable:BTC
+
+# Labels: income
+
 if %direction ^IN$
 & %label (Income|Payment)
  account2 income:electrum:%subaccount:BTC
-  comment txid:%txid, notes:%label, direction:%direction, taxed_as:INCOME
-  comment2 %timestamp,INCOME,electrum:%subaccount:BTC,BTC,%value_without_fee,USD,%fiat_value,%label
+  comment txid:%oc_hash, notes:%label, direction:%direction, taxed_as:INCOME
+  comment2 %timestamp,INCOME,electrum:%subaccount:BTC,BTC,%oc_amount,USD,%fiat_value,%label
+
+if %direction ^IN$
+& %label (Income|Payment)
+& %ln_amount [1-9]
+  comment ln_hash:%ln_hash, notes:%label, direction:%direction, taxed_as:INCOME
+  comment2 %timestamp,INCOME,electrum:%subaccount:BTC,BTC,%ln_amount,USD,%fiat_value,%label
+
+# Labels: gifts
+# NOTE: fiat_value must be manually updated in custom rules if cost-basis will be different than given fiat_value
+
+# TODO: add regex for donate/donation
+# TODO: separate donations from gifts subaccount
+if %direction ^IN$
+& %label Gift
+& %oc_amount [1-9]
+  account2 income:electrum:%subaccount:gifts:BTC
+  comment txid:%oc_hash, notes:%label, direction:%direction, taxed_as:GIFTIN
+  comment2 %timestamp,GIFTIN,electrum:%subaccount:BTC,BTC,%oc_amount,USD,%fiat_value,%label

 if %direction ^IN$
 & %label Gift
-  account2 income:electrum:%subaccount:gifts:BTC
-  comment txid:%txid, notes:%label, direction:%direction, taxed_as:GIFTIN
-  comment2 %timestamp,GIFTIN,electrum:%subaccount:BTC,BTC,%value_without_fee,USD,%fiat_value,%label
-# NOTE: fiat_value must be manually updated in custom rules if your given cost-basis will be different
+& %ln_amount [1-9]
+  comment ln_hash:%ln_hash, notes:%label, direction:%direction, taxed_as:GIFTIN
+  comment2 %timestamp,GIFTIN,electrum:%subaccount:BTC,BTC,%ln_amount,USD,%fiat_value,%label

+#
 # OUT
+#

 if %direction ^OUT$
+  amount -%oc_amount BTC
  account2 expenses:electrum:%subaccount:BTC
-  comment txid:%txid, notes:%label, taxed_as:SPEND
-  comment2 %timestamp,SPEND,electrum:%subaccount:BTC,BTC,%value_without_fee,USD,%fiat_value,%label
+  comment txid:%oc_hash, notes:%label, taxed_as:SPEND
+  comment2 %timestamp,SPEND,electrum:%subaccount:BTC,BTC,%oc_amount,USD,%fiat_value,%label

 if %direction ^OUT$
-& %label (Self|Equity transfer|Capital contribution)
-  account2 equity:electrum:%subaccount:deposit:BTC
-  comment txid:%txid, notes:%label, direction:%direction
-  comment2
-# NOTE: custom rule equity transfers should also clear comment2
+& %ln_amount [1-9]
+  comment txid:%oc_hash, ln_hash:%ln_hash, notes:%label, direction:%direction, taxed_as:SPEND
+  comment2 %timestamp,SPEND,electrum:%subaccount:BTC,BTC,%ln_amount,USD,%fiat_value,%label

+if %direction ^OUT$
+& %oc_amount ^[^1-9]*$
+& %ln_amount [1-9]
+  amount -%ln_amount BTC
+  amount2 %ln_amount BTC
+
+# Label: lightning channels
+
+if %direction ^OUT$
+& %label ^Open channel
+& %oc_amount [1-9]
+& %ln_amount [1-9]
+  amount -%ln_amount BTC
+  account2 assets:electrum:%subaccount:ln_locked_payable:BTC
+  comment2
+
+if %direction ^OUT$
+& %label (^sweep local_anchor|^sweep our_ctx_to_local)
+& %oc_amount [1-9]
+& %ln_amount ^[^1-9]*$
+  account2 assets:electrum:%subaccount:BTC
+  comment
+  comment2
+
+# Treat this OUT as an IN as it's unlocking
+if %direction ^OUT$
+& %label ^Close channel
+& %oc_hash ^[^a-z0-9]*$
+& %ln_hash ^[^a-z0-9]*$
+& %oc_amount ^[^1-9]*$
+& %ln_amount [1-9]
+  amount1 -%ln_amount BTC
+  account1 assets:electrum:%subaccount:ln_locked_payable:BTC
+  amount2 %ln_amount BTC
+  account2 assets:electrum:%subaccount:BTC
+  comment notes:%label, direction:%direction
+  comment2
+
+if %direction ^OUT$
+& %label ^Close channel
+& %oc_amount [1-9]
+& %ln_amount [1-9]
+  amount1 -%ln_amount BTC
+  account1 assets:electrum:%subaccount:ln_locked_payable:BTC
+  amount2 %ln_amount BTC
+  account2 assets:electrum:%subaccount:BTC
+  comment2
+
+# Label: transfers
+# NOTE: custom rule equity transfers should also clear comment2
+if %direction ^OUT$
+& %label (Self|Transfer|Equity transfer|Capital contribution)
+  account2 equity:electrum:%subaccount:deposit:BTC
+  comment txid:%oc_hash, notes:%label, direction:%direction
+  comment2
+
+if %direction ^OUT$
+& %label (Self|Transfer|Equity transfer|Capital contribution)
+& %ln_amount [1-9]
+  comment txid:%oc_hash, ln_hash:%ln_hash, notes:%label, direction:%direction
+
+# Label: gifts
+# TODO: add regex for donate/donation
+# TODO: separate donations from gifts subaccount
 if %direction ^OUT$
 & %label Gift
  account2 expenses:electrum:%subaccount:gifts:BTC
-  comment txid:%txid, notes:%label, direction:%direction, taxed_as:GIFT
-  comment2 %timestamp,GIFT,electrum:%subaccount:BTC,BTC,%value_without_fee,USD,%fiat_value,%label
+  comment txid:%oc_hash, notes:%label, direction:%direction, taxed_as:GIFT
+  comment2 %timestamp,GIFT,electrum:%subaccount:BTC,BTC,%oc_amount,USD,%fiat_value,%label

-# Fees
 if %direction ^OUT$
-& %fee [1-9]
+& %label Gift
+& %ln_amount [1-9]
+  comment txid:%oc_hash, ln_hash:%ln_hash, notes:%label, direction:%direction, taxed_as:GIFT
+  comment2 %timestamp,GIFT,electrum:%subaccount:BTC,BTC,%ln_amount,USD,%fiat_value,%label
+
+#
+# Fees
+#
+
+if %direction ^OUT$
+& %network_fee [1-9]
  account3 assets:electrum:%subaccount:BTC
-  amount3 -%fee BTC
+  amount3 -%network_fee BTC
  account4 expenses:electrum:%subaccount:fees:BTC
-  amount4 %fee BTC
-  comment txid:%txid, notes:%label, direction:%direction, taxed_as:SPEND
-  comment3 %timestamp,SPEND,electrum:%subaccount:BTC,BTC,%fee,USD,%fiat_fee,FEE
+  amount4 %network_fee BTC
+  comment txid:%oc_hash, notes:%label, direction:%direction, taxed_as:SPEND
+  comment3 %timestamp,SPEND,electrum:%subaccount:BTC,BTC,%network_fee,USD,%fiat_fee,FEE
 # Using comment3 so a comment2 SPEND isn't overwritten (when applicable)

 # vim: sw=2 sts=2 si ai et