#!/usr/bin/env bash # docker-finance | modern accounting for the power-user # # Copyright (C) 2021-2024 Aaron Fiore (Founder, Evergreen Crypto LLC) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # "Libraries" # [ -z "$DOCKER_FINANCE_CONTAINER_REPO" ] && exit 1 source "${DOCKER_FINANCE_CONTAINER_REPO}/src/hledger-flow/lib/lib_preprocess.bash" "$1" "$2" # # Implementation # [ -z "$global_year" ] && exit 1 [ -z "$global_subaccount" ] && exit 1 [ -z "$global_in_path" ] && exit 1 [ -z "$global_out_path" ] && exit 1 [ -z "$global_in_filename" ] && exit 1 # NOTES: # # - Coinbase REST API CSVs are a beast. They provide inconsistent headers and # are complex. # # - When fetching new files, if you are expecting no changes, the transaction # confirmation number may change and subsequently change your git-diff # # TODO: optimize function parse() { # # Create account ID as first column, followed by TXID # # NOTE: *MUST* have filename with format, where X is account ID: # # XXXXXXXXXXXXX-XXXX-XXXX-XXXXXXXXXXXX_transactions.csv # local _account_id="${global_in_filename:0:-17}" local _details_header="details_header2" # courtesy of fix_header() local _header="id,type,status,amount_amount,amount_currency,native_amount_amount,native_amount_currency,description,created_at,updated_at,resource,resource_path,instant_exchange,details_title,details_subtitle,${_details_header},network_transaction_fee_amount,network_transaction_fee_currency,network_transaction_amount_amount,network_transaction_amount_currency,to_currency,to_address_info_address,advanced_trade_fill_fill_price,advanced_trade_fill_product_id,advanced_trade_fill_order_id,advanced_trade_fill_commission,advanced_trade_fill_order_side,network_hash,network_status" # A clean but hacky way to keep the stream intact while testing headers local _header_network="network_transaction_fee_amount,network_transaction_fee_currency,network_transaction_amount_amount,network_transaction_amount_currency,network_hash,network_status" if lib_preprocess::test_header "$_header_network"; then _header_network=""; fi local _header_to="to_currency,to_address_info_address" if lib_preprocess::test_header "$_header_to"; then _header_to=""; fi local _header_advanced="advanced_trade_fill_fill_price,advanced_trade_fill_product_id,advanced_trade_fill_order_id,advanced_trade_fill_commission,advanced_trade_fill_order_side" if lib_preprocess::test_header "$_header_advanced"; then _header_advanced=""; fi # NOTE: csvjoin BUG(?): csvjoin *MUST* use -I or else satoshis like 0.00000021 will turn into 21000000 BTC! csvjoin -I --snifflimit 0 "$global_in_path" <(fix_header "$global_in_path") \ | if [ ! -z "$_header_network" ]; then add_to_header - "$_header_network"; else cat; fi \ | if [ ! -z "$_header_to" ]; then add_to_header - "$_header_to"; else cat; fi \ | if [ ! -z "$_header_advanced" ]; then add_to_header - "$_header_advanced"; else cat; fi \ | xsv select "$_header" \ | sed -e "s:^:${_account_id},:g" -e "1 s:^${_account_id},:account_id,:g" \ | __parse >"$global_out_path" # TODO: complete hack because hledger-flow won't ignore deleted 1-in and # 2-preprocessed files after script has been called. Is this still needed? # # NOTES: # - Must work/be noted in rules file. # - Parsing correct years into correct directories can create empty files # in 2-preprocessed because the innapropriate data is sent to 1-in to # begin with. This can possibly be handled in the fetching API but it's # easier to do here for now. if [[ ! -s "$global_out_path" ]]; then echo "SKIP,SKIP,SKIP,000.00000000,SKIP,00000.00,SKIP,SKIP,0000-00-00 00:00:00,0000-00-00 00:00:00,SKIP,SKIP,SKIP,SKIP,SKIP,SKIP,0.00000000,SKIP,000.00000000,SKIP,00.00000000,0.00000000,0.00000000" \ >"$global_out_path" fi } # Fixes column data that has comma (which will ruin output order) function fix_header() { # NOTE: output will be details_header2, not details_header xsv select "details_header" "$1" \ | sed -e 's:,::g' -e 's:"::g' } # Adds column to CSV function add_to_header() { csvjoin -I --snifflimit 0 "$1" <(echo "$2") } # Primary parse impl after previous preparation function __parse() { gawk -v global_year="$global_year" -v global_subaccount="$global_subaccount" \ '{ if (NR<2 || $10 !~ global_year) next # Coinbase is not uniform in its - sign, so remove it and deal re-add in rules direction=($5 ~ /^-/ ? "OUT" : "IN") sub(/^-/, "", $5) sub(/^-/, "", $7) # Cleanup the time sub(/T/, " ", $10); sub(/T/, " ", $11) sub(/Z/, "", $10); sub(/Z/, "", $11) sub(/\+.*/, "", $10); sub(/\+.*/, "", $11) # Print printf $1 OFS # account id printf $2 OFS # id (coinbase_id) printf $3 OFS # type printf $4 OFS # status printf $5 OFS # amount_amount printf $6 OFS # amount_currency printf $7 OFS # native_amount_amount printf $8 OFS # native_amount_currency printf $9 OFS # description printf $10 OFS # created_at printf $11 OFS # updated_at printf $12 OFS # resource printf $13 OFS # resource_path printf $14 OFS # instant_exchange printf $15 OFS # details_title printf $16 OFS # details_subtitle printf $17 OFS # details_header2 printf("%.8f", $18); printf OFS # network_transaction_fee_amount printf $19 OFS # network_transaction_fee_currency printf("%.8f", $20); printf OFS # network_transaction_amount_amount printf $21 OFS # network_transaction_amount_currency printf $22 OFS # to_currency printf $23 OFS # to_address_info_address # Add new columns to calculate fees against native currency price # NOTE: 0 because CB does not accurately display fiat amount # if satoshi is small (if less than $0.00) if ($7 > 0) {printf("%.8f", $7 / $5)} printf OFS # native_amount_price if ($7 > 0) {printf("%.8f", ($7 / $5) * $18)} printf OFS # native_network_transaction_fee_amount if ($7 > 0) {printf("%.8f", ($5 * 0.01) / $7)} printf OFS # native_conversion_fee_amount # NOTE: this is a guestimation of actual conversion fees because Coinbase # never specifies exact amount. # CB Advanced Trading printf("%.8f", $24); printf OFS # advanced_trade_fill_fill_price printf $25 OFS # advanced_trade_fill_product_id printf $26 OFS # advanced_trade_fill_order_id printf("%.8f", $27); printf OFS # advanced_trade_fill_commission printf $28 OFS # advanced_trade_fill_order_side # USER ADDED: small_satoshi_multiplier (for sat trades less than a penny, # since `@@ 0` is not needed (or any empty tax columns)) if ($7 == 0) {printf("%.8f", $24 * $5)}; printf OFS printf $29 OFS # network_hash (txid, if applicable) printf $30 OFS # network_status (if applicable) printf direction OFS printf global_subaccount printf "\n"; }' FS=, OFS=, } function main() { parse } main "$@" # vim: sw=2 sts=2 si ai et