From 5352e341968d2aabd14f0ddfc59f29352cbd183f Mon Sep 17 00:00:00 2001 From: Aaron Fiore Date: Thu, 6 Feb 2025 18:40:38 -0800 Subject: [PATCH] hledger-flow: btcpayserver: add prelim support for v2 reports - Removes previous format (fully deprecated) - Adds support for "Legacy Invoice" and "Wallets" reports See code notes regarding caveats and TODOs. --- .../btcpayserver/btcpayserver-shared.bash | 199 ++++++++++++++++-- .../btcpayserver/btcpayserver-shared.rules | 67 ++++-- 2 files changed, 224 insertions(+), 42 deletions(-) diff --git a/container/src/hledger-flow/accounts/btcpayserver/btcpayserver-shared.bash b/container/src/hledger-flow/accounts/btcpayserver/btcpayserver-shared.bash index a87b9dd..cb74e17 100755 --- a/container/src/hledger-flow/accounts/btcpayserver/btcpayserver-shared.bash +++ b/container/src/hledger-flow/accounts/btcpayserver/btcpayserver-shared.bash @@ -2,7 +2,7 @@ # docker-finance | modern accounting for the power-user # -# Copyright (C) 2021-2024 Aaron Fiore (Founder, Evergreen Crypto LLC) +# Copyright (C) 2021-2025 Aaron Fiore (Founder, Evergreen Crypto LLC) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,6 +23,7 @@ [ -z "$DOCKER_FINANCE_CONTAINER_REPO" ] && exit 1 source "${DOCKER_FINANCE_CONTAINER_REPO}/src/hledger-flow/lib/lib_preprocess.bash" "$1" "$2" +source "${DOCKER_FINANCE_CONTAINER_REPO}/src/hledger-flow/lib/lib_utils.bash" || exit 1 # # Implementation @@ -34,45 +35,197 @@ source "${DOCKER_FINANCE_CONTAINER_REPO}/src/hledger-flow/lib/lib_preprocess.bas [ -z "$global_in_path" ] && exit 1 [ -z "$global_out_path" ] && exit 1 -function parse() +function btcpayserver::print_warning() { - lib_preprocess::assert_header "Transaction Id,Timestamp,Amount,Currency,Is Confirmed,Comment,Labels" + [ -z "$global_account" ] && lib_utils::die_fatal + lib_utils::print_warning "${global_account}: '$1' report is not supported" +} +# +# BTCPay Server v2 +# + +# NOTE/TODO: +# +# TL;DR: +# +# - *MUST* export both "Legacy Invoice" report and "Wallets" report for an accurate balance. +# * The "Legacy Invoice" report currently does *not* include any outgoing txs (refunds or transfers). +# +# WARNING: +# +# - All outgoing txs do *not* account for fees (they are lumped in with the total). +# * Until upstream changes this, any fees must be separated manually w/ custom rules. +# +# CAUTION: +# +# - The "Wallets" report contains the txid that the "Payments" report *should* include (but doesn't). +# * The "Legacy Invoice" report provides both. +# +# - The "Wallets" report does *not* contain a description for refunds versus transfers. +# * This is done in "Payouts" report or "Refunds" report. +# +# - The "Payouts" report contains the destination address whereas "Refunds" does not. +# * "Payouts" places the Invoice ID in the 'Source' field. +# * "Refunds" places the Invoice ID in the 'InvoiceId' field. + +# "Legacy Invoice" report +function btcpayserver::legacy() +{ gawk -v global_year="$global_year" -v global_subaccount="$global_subaccount" \ '{ - if (NR<2 || $2 !~ global_year) + if (NR<2 || $1 !~ global_year) next - # TODO: multi-line comments will break (they literally split the comment across multiple newlines...) - if ($0 ~ /","/) - skip + printf $1 OFS # ReceivedDate + printf $2 OFS # StoreId + printf $3 OFS # OrderId + printf $4 OFS # InvoiceId + printf $5 OFS # InvoiceCreatedDate + printf $6 OFS # InvoiceExpirationDate + printf $7 OFS # InvoiceMonitoringDate - # Format: MM/DD/YYYY HH:MM:SS +00:00 - timestamp=substr($2, 1, 19) # Remove tail - sub(/ /, "T", timestamp) # Remove space + # PaymentId + # NOTE: BTCPay Server will append the block index as "-N" to the txid + txid=substr($8, 1, 64); printf txid OFS + ind=substr($8, 66, 2); printf ind OFS - # Get/set date format - cmd = "date \"+%F %T %z\" --date="timestamp | getline date + printf $9 OFS # Destination + printf $10 OFS # PaymentType + printf $11 OFS # CryptoCode - printf $1 OFS # Transaction Id - printf date OFS # Timestamp - printf $3 OFS # Amount - printf $4 OFS # Currency - printf $5 OFS # Is Confirmed - printf OFS # Comment TODO: column allows commas and multi-line comments - printf OFS # Labels TODO: column allows commas + # Paid + printf("%.8f", $12); printf OFS - printf ($3 ~ /^-/ ? "OUT" : "IN") OFS # Direction - printf global_subaccount + printf $13 OFS # NetworkFee + printf $14 OFS # ConversionRate + printf $15 OFS # PaidCurrency + printf $16 OFS # InvoiceCurrency + printf $17 OFS # InvoiceDue + printf $18 OFS # InvoicePrice + printf $19 OFS # InvoiceItemCode + + # TODO: if description contains comma(s)? + printf $20 OFS # InvoiceItemDesc + + printf $21 OFS # InvoiceFullStatus + printf $22 OFS # InvoiceStatus + printf $23 OFS # InvoiceExceptionStatus + printf $24 OFS # BuyerEmail + printf $25 OFS # Accounted + + # WARNING: appears to be always IN (see notes regarding "Wallets" report) + printf "IN" OFS # Direction + printf global_subaccount # Subaccount printf "\n" + }' FS=, OFS=, "$global_in_path" +} - }' FS=, OFS=, "$global_in_path" >"$global_out_path" +# "Payments" report +function btcpayserver::payments() +{ + btcpayserver::print_warning "Payments" +} + +# "Payouts" report +function btcpayserver::payouts() +{ + btcpayserver::print_warning "Payouts" +} + +# "Refunds" report +function btcpayserver::refunds() +{ + btcpayserver::print_warning "Refunds" +} + +# "Sales" report +function btcpayserver::sales() +{ + btcpayserver::print_warning "Sales" + # TODO: Accounting for this needs more consideration, + # as these are individual products sold w/ only fiat value given. +} + +# "Wallets" report +function btcpayserver::wallets() +{ + gawk -v global_year="$global_year" -v global_subaccount="$global_subaccount" \ + '{ + if (NR<2 || $1 !~ global_year) + next + + # All "IN"s are removed and handled by legacy invoice + if ($6 !~ /^-/) + next + + printf $1 OFS # Date (ReceivedDate) + printf OFS # (StoreId) + printf OFS # (OrderId) + printf $4 OFS # InvoiceId (InvoiceId) + printf OFS # (InvoiceCreatedDate) + printf OFS # (InvoiceExpirationDate) + printf OFS # (InvoiceMonitoringDate) + + # (PaymentId) + printf $3 OFS # TransactionId (txid) + printf OFS # (Block Index) + + printf OFS # (Destination) + printf OFS # (PaymentType) + printf $2 OFS # Crypto (CryptoCode) + + # BalanceChange (Paid) + printf("%.8f", $6); printf OFS + + printf OFS # (NetworkFee) + printf OFS # (ConversionRate) + printf OFS # (PaidCurrency) + printf OFS # (InvoiceCurrency) + printf OFS # (InvoiceDue) + printf OFS # (InvoicePrice) + printf OFS # (InvoiceItemCode) + printf OFS # (InvoiceItemDesc) + printf OFS # (InvoiceFullStatus) + printf OFS # (InvoiceStatus) + printf OFS # (InvoiceExceptionStatus) + printf OFS # (BuyerEmail) + printf $5 OFS # Confirmed (Accounted) + + # All "IN"s are removed and handled by legacy invoice + printf "OUT" OFS # Direction + printf global_subaccount # Subaccount + + printf "\n" + }' FS=, OFS=, "$global_in_path" +} + +function btcpayserver::parse() +{ + lib_preprocess::test_header "ReceivedDate,StoreId,OrderId,InvoiceId,InvoiceCreatedDate,InvoiceExpirationDate,InvoiceMonitoringDate,PaymentId,Destination,PaymentType,CryptoCode,Paid,NetworkFee,ConversionRate,PaidCurrency,InvoiceCurrency,InvoiceDue,InvoicePrice,InvoiceItemCode,InvoiceItemDesc,InvoiceFullStatus,InvoiceStatus,InvoiceExceptionStatus,BuyerEmail,Accounted" \ + && btcpayserver::legacy + + lib_preprocess::test_header "Date,InvoiceId,OrderId,Category,PaymentMethodId,Confirmed,Address,PaymentCurrency,PaymentAmount,PaymentMethodFee,LightningAddress,InvoiceCurrency,InvoiceCurrencyAmount,Rate" \ + && btcpayserver::payments + + lib_preprocess::test_header "Date,Source,State,PaymentType,Currency,Amount,OriginalCurrency,OriginalAmount,Destination" \ + && btcpayserver::payouts + + lib_preprocess::test_header "Date,InvoiceId,Currency,Completed,Awaiting,Limit,FullyPaid" \ + && btcpayserver::refunds + + lib_preprocess::test_header "Date,InvoiceId,State,AppId,Product,Quantity,CurrencyAmount,Currency" \ + && btcpayserver::sales + + lib_preprocess::test_header "Date,Crypto,TransactionId,InvoiceId,Confirmed,BalanceChange" \ + && btcpayserver::wallets } function main() { - parse + # WARNING: upstream produces carriage return all EOL + btcpayserver::parse | sed -e 's:\x0d::g' >"$global_out_path" || lib_utils::catch $? } main "$@" diff --git a/container/src/hledger-flow/accounts/btcpayserver/btcpayserver-shared.rules b/container/src/hledger-flow/accounts/btcpayserver/btcpayserver-shared.rules index 7185b49..18ec31b 100644 --- a/container/src/hledger-flow/accounts/btcpayserver/btcpayserver-shared.rules +++ b/container/src/hledger-flow/accounts/btcpayserver/btcpayserver-shared.rules @@ -1,6 +1,6 @@ # docker-finance | modern accounting for the power-user # -# Copyright (C) 2021-2024 Aaron Fiore (Founder, Evergreen Crypto LLC) +# Copyright (C) 2021-2025 Aaron Fiore (Founder, Evergreen Crypto LLC) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,38 +15,67 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -fields txid,timestamp,amount_,currency_,is_confirmed,comment,labels,direction,subaccount +# TODO: currently, only supports "Legacy Invoice" and "Wallets" format (w/ additional custom columns) +fields ReceivedDate,StoreId,OrderId,InvoiceId,InvoiceCreatedDate,InvoiceExpirationDate,InvoiceMonitoringDate,txid,index,Destination,PaymentType,CryptoCode,Paid,NetworkFee,ConversionRate,PaidCurrency,InvoiceCurrency,InvoiceDue,InvoicePrice,InvoiceItemCode,InvoiceItemDesc,InvoiceFullStatus,InvoiceStatus,InvoiceExceptionStatus,BuyerEmail,Accounted,direction,subaccount # NOTE: BTCPayServer exports to localtime -date-format %Y-%m-%d %H:%M:%S %z -date %timestamp +date-format %Y-%m-%d %H:%M:%S +date %ReceivedDate -description %timestamp +description %ReceivedDate -comment txid:%txid, direction:%direction +if %InvoiceItemDesc [a-z0-9] + description %ReceivedDate | %InvoiceItemDesc -if %comment [a-z0-9] - comment txid:%txid, comment:%comment, direction:%direction +account1 assets:btcpayserver:%subaccount:%CryptoCode +amount %Paid %CryptoCode -if %labels [a-z0-9] - comment txid:%txid, labels:%labels, direction:%direction +# If invoice is expired or otherwise not accounted for, skip +if %Accounted ^[^a-z]*$ + skip -if %comment [a-z0-9] -& %labels [a-z0-9] - comment txid:%txid, comment:%comment, labels:%labels, direction:%direction +# +# Comment +# -account1 assets:btcpayserver:%subaccount:%currency_ -amount %amount_ %currency_ +# Default comment +comment created:%InvoiceCreatedDate, expired:%InvoiceExpirationDate, store_id:%StoreId, order_id:%OrderId, invoice_id:%InvoiceId, type:%PaymentType, to_address:%Destination, txid:%txid, index:%index, status:%InvoiceStatus, direction:%direction + +# Comment w/ item code +if %InvoiceItemCode [a-z0-9] + comment created:%InvoiceCreatedDate, expired:%InvoiceExpirationDate, store_id:%StoreId, order_id:%OrderId, invoice_id:%InvoiceId, item_code:%InvoiceItemCode, type:%PaymentType, to_address:%Destination, txid:%txid, index:%index, status:%InvoiceStatus, direction:%direction + +# Comment w/ buyer email +if %BuyerEmail [a-z0-9] + comment created:%InvoiceCreatedDate, expired:%InvoiceExpirationDate, store_id:%StoreId, order_id:%OrderId, invoice_id:%InvoiceId, email:%BuyerEmail, type:%PaymentType, to_address:%Destination, txid:%txid, index:%index, status:%InvoiceStatus, direction:%direction + +# Comment w/ both item code + buyer email +if %InvoiceItemCode [a-z0-9] +& %BuyerEmail [a-z0-9] + comment created:%InvoiceCreatedDate, expired:%InvoiceExpirationDate, store_id:%StoreId, order_id:%OrderId, invoice_id:%InvoiceId, item_code:%InvoiceItemCode, email:%BuyerEmail, type:%PaymentType, to_address:%Destination, txid:%txid, index:%index, status:%InvoiceStatus, direction:%direction + +# Comment for "Wallets" export +if %direction ^OUT$ + comment txid:%txid, confirmed:%Accounted, direction:%direction + +# +# Direction +# # Default income if %direction ^IN$ - account2 income:btcpayserver:%subaccount:%currency_ + account2 income:btcpayserver:%subaccount:%CryptoCode + comment2 %ReceivedDate,INCOME,btcpayserver:%subaccount,%CryptoCode,%Paid,%InvoiceCurrency,%PaidCurrency,%InvoiceId # Default equity transfer if %direction ^OUT$ - account2 equity:btcpayserver:%subaccount:deposit:%currency_ + account2 equity:btcpayserver:%subaccount:deposit:%CryptoCode -# TODO: add caveat for refunds -# TODO: upstream doesn't provide fees in the exported CSV! +# TODO: +# +# WARNING: +# +# - All outgoing txs do *not* account for fees (they are lumped in with the total). +# * Until upstream changes this, any fees must be separated manually w/ custom rules. # vim: sw=2 sts=2 si ai et