EvergreenCrypto/docker-finance
1
1
Fork 1
Code Issues 31 Pull Requests 2 Actions Releases 10 Activity

container: lib_fetch: refactor to use yq

Browse Source 
Replaces `shyaml` dependency with `yq`.
This commit is contained in:
Aaron Fiore
2024-08-17 17:14:41 -07:00
parent 1402284f2b
commit a3b889052b
1 changed files with 92 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
container/src/finance/lib/internal/lib_fetch.bash
View File

@@ -352,6 +352,7 @@ function lib_fetch::__parse_args()
fi fi
} }
# TODO: complete __fetch() rewrite
function lib_fetch::__fetch() function lib_fetch::__fetch()
{ {
# Supported remote fetch accounts # Supported remote fetch accounts
@@ -368,15 +369,17 @@ function lib_fetch::__fetch()
"pera-wallet" "pera-wallet"
) )
[ -z "$global_parent_profile" ] && lib_utils::die_fatal
[ -z "$global_child_profile" ] && lib_utils::die_fatal
# TODO: global args should be set in arg parsing and made readonly # TODO: global args should be set in arg parsing and made readonly
# Fetch only given accounts # Fetch only given accounts
if [ ! -z "$global_arg_account" ]; then if [ ! -z "$global_arg_account" ]; then
for _account in "${global_arg_account[@]}"; do for _account in "${global_arg_account[@]}"; do
if ! lib_fetch::__parse_yaml "get-value" "account.${_account}" 1 &>/dev/null; then local _value
# Unsupported or unavailable _value="$(lib_fetch::__parse_yaml "get-value" "account.${_account}" 2>/dev/null)"
[ -z "$global_parent_profile" ] && lib_utils::die_fatal if [[ "$_value" == "null" ]]; then
[ -z "$global_child_profile" ] && lib_utils::die_fatal lib_utils::print_warning "account.${_account} not found, skipping!"
lib_utils::print_warning "${global_parent_profile}.${global_child_profile}.account.${_account} not found, skipping!"
else else
local _accounts+=("$_account") local _accounts+=("$_account")
fi fi
@@ -442,39 +445,49 @@ function lib_fetch::__fetch_account()
local _members=("key" "passphrase" "secret" "subaccount") local _members=("key" "passphrase" "secret" "subaccount")
for _member in "${_members[@]}"; do for _member in "${_members[@]}"; do
# Get member value # Get member value
if _value=$(lib_fetch::__parse_yaml "get-value" "account.${_account}.${_member}"); then local _value
_value=$(lib_fetch::__parse_yaml "get-values" "account.${_account}.${_member}")
# Member value for key if [ $? -eq 0 ]; then
if [[ "$_member" == "key" ]]; then
local _api_key="$_value" # Sanitize for 'yq'-specific caveats
if [[ -z "$_api_key" && $_need_key == true ]]; then local _sanitized
lib_utils::die_fatal "$_account - empty fetch ${_member}!" if [[ "$_value" == "null" ]]; then
fi _sanitized=""
else
# Values may contain 'yq'-interpreted characters (newlines, etc.)
# NOTE: this is needed for at least Coinbase's CDP secret
_sanitized="$(echo "$_value" | sed -e 's:^[ \t]*::' -e '/^$/d' -e "s:^'::" -e "s:'$::")"
fi fi
# Member value for passphrase case "$_member" in
if [[ "$_member" == "passphrase" ]]; then key)
local _api_passphrase="$_value" local _api_key="$_sanitized"
if [[ -z "$_api_passphrase" && $_need_passphrase == true ]]; then [[ -z "$_api_key" && $_need_key == true ]] \
lib_utils::die_fatal "$_account - empty fetch ${_member}!" && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
fi ;;
fi passphrase)
local _api_passphrase="$_sanitized"
# Member value for secret [[ -z "$_api_passphrase" && $_need_passphrase == true ]] \
if [[ "$_member" == "secret" ]]; then && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
local _api_secret="$_value" ;;
if [[ -z "$_api_secret" && $_need_secret == true ]]; then secret)
lib_utils::die_fatal "$_account - empty fetch ${_member}!" local _api_secret="$_sanitized"
fi [[ -z "$_api_secret" && $_need_secret == true ]] \
fi && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
;;
# Member value for subaccount (will *always* need subaccount) subaccount)
if [[ "$_member" == "subaccount" ]]; then # NOTE: member value for subaccount will *always* need subaccount
local _api_subaccount="$_value" local _api_subaccount="$_sanitized"
[ -z "$_api_subaccount" ] \ [ -z "$_api_subaccount" ] \
&& lib_utils::die_fatal "$_account - empty fetch ${_member}!" && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
fi ;;
*)
lib_utils::die_fatal "Unsupported member"
;;
esac
else else
lib_utils::print_warning "$_account is unavailable, skipping!" lib_utils::print_warning "$_account is unavailable, skipping!"
continue 2 continue 2
@@ -494,6 +507,7 @@ function lib_fetch::__fetch_account()
# (we'll pass entire string to internal fetch impl) # (we'll pass entire string to internal fetch impl)
# TODO: parse/cut before setting var # TODO: parse/cut before setting var
[ -z "$global_child_profile_flow" ] && lib_utils::die_fatal [ -z "$global_child_profile_flow" ] && lib_utils::die_fatal
local _sub local _sub
_sub="$(echo $_api_subaccount | cut -d'/' -f1)" _sub="$(echo $_api_subaccount | cut -d'/' -f1)"
local _api_out_dir="${global_child_profile_flow}/import/${global_child_profile}/${_account}/${_sub}/1-in/${global_arg_year}" local _api_out_dir="${global_child_profile_flow}/import/${global_child_profile}/${_account}/${_sub}/1-in/${global_arg_year}"
@@ -508,9 +522,8 @@ function lib_fetch::__fetch_account()
# TODO: clarify, add more/better comments # TODO: clarify, add more/better comments
# If subaccount is type 'struct', then subaccount will be list of addresses # If subaccount is list of addresses, create 'blockchain:address' format (for internal fetch impl)
# So, create what's needed for internal fetch impl (blockchain:address format) if echo "$_api_subaccount" | head -n2 | grep ":*$" | tail -n1 | grep -q "^- "; then
if [[ $(lib_fetch::__parse_yaml "get-type" "account.${_account}.subaccount") == "struct" ]]; then
# Parse out the separate blockchains # Parse out the separate blockchains
local _blockchain local _blockchain
@@ -520,7 +533,7 @@ function lib_fetch::__fetch_account()
_parsed_csv=$( _parsed_csv=$(
echo "$_blockchain" | while read _key; do echo "$_blockchain" | while read _key; do
local _addresses local _addresses
_addresses=$(echo "$_api_subaccount" | shyaml get-value "$_key") # TODO: use lib _addresses=$(echo "$_api_subaccount" | yq -M -y --indentless -e ".${_key}") # TODO: use lib
echo "$_addresses" | while read _value; do echo "$_addresses" | while read _value; do
echo "${_key}:${_value}" | sed -e 's/:- /\//g' -e "s:'::g" echo "${_key}:${_value}" | sed -e 's/:- /\//g' -e "s:'::g"
done done
@@ -663,20 +676,24 @@ function lib_fetch::__fetch_price()
local _members=("key" "asset") local _members=("key" "asset")
for _member in "${_members[@]}"; do for _member in "${_members[@]}"; do
if _value=$(lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}"); then
if [[ "$_member" == "key" ]]; then local _value
[ -z "$_value" ] \ _value="$(lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}" 2>/dev/null)"
&& lib_utils::print_warning "${_api} has empty ${_member}"
fi case "$_member" in
key)
[[ "$_value" == "null" ]] && lib_utils::print_warning "${_api} has empty ${_member}"
;;
asset)
if ! lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}" | grep -q "^- "; then
lib_utils::die_fatal "Invalid asset sequence!"
fi
;;
*)
lib_utils::die_fatal "${_api} has invalid member: ${_member}"
;;
esac
if [[ "$_member" == "asset" ]]; then
[[ $(lib_fetch::__parse_yaml "get-type" "price.${_api}.${_member}") != "sequence" ]] \
&& lib_utils::die_fatal "Invalid asset sequence!"
fi
else
lib_utils::die_fatal "'${_member}' is unavailable"
fi
done done
# Available API(s) # Available API(s)
@@ -707,7 +724,7 @@ function lib_fetch::__fetch_price()
# Parse out each asset # Parse out each asset
while read _line; do while read _line; do
local _assets+=("$_line") local _assets+=("$_line")
done < <(lib_fetch::__parse_yaml "get-values" "price.${_api}.asset") done < <(lib_fetch::__parse_yaml "get-values" "price.${_api}.asset" | sed 's:^- ::')
else else
# Get assets from CLI # Get assets from CLI
local _assets=("${global_arg_price[*]}") local _assets=("${global_arg_price[*]}")
@@ -766,14 +783,34 @@ function lib_fetch::__fetch_price()
function lib_fetch::__parse_yaml() function lib_fetch::__parse_yaml()
{ {
[[ -z "$1" || -z "$2" ]] && lib_utils::die_fatal
local _action="$1" local _action="$1"
local _append="$2" local _append="$2"
[ -z "$_action" ] && lib_utils::die_fatal
[ -z "$_append" ] && lib_utils::die_fatal
[ -z "$global_conf_fetch" ] && lib_utils::die_fatal [ -z "$global_conf_fetch" ] && lib_utils::die_fatal
echo "$(<$global_conf_fetch)" \ local -r _raw=".${global_parent_profile}.${global_child_profile}.${_append}"
| shyaml -q "$_action" "${global_parent_profile}"."${global_child_profile}"."${_append}"
# `yq` (kislyuk's) requires quotes around keys/members that contain '-'
local _arg="$_raw"
[[ "$_arg" =~ '-' ]] && _arg="$(sed -e 's:\.:"\.":g' -e 's:"\.:\.:' -e "s: *$:\":" <(echo "$_raw"))"
# Pseudo-substitution of `shyaml` functionality
local _ifs="$IFS"
IFS=' '
local -r _cmd=("yq" "-M" "-y" "--indentless" "-e" "$_arg" "$global_conf_fetch")
local -r _grep=("grep" "-v" "^\.\.\.$")
case "$_action" in
get-value)
"${_cmd[@]}" | "${_grep[@]}" -m2
;;
get-values)
"${_cmd[@]}" | "${_grep[@]}"
;;
*)
lib_utils::die_fatal "Unsupported YAML action"
;;
esac
IFS="$_ifs"
} }
function lib_fetch::__fetch_exec() function lib_fetch::__fetch_exec()