diff --git a/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in b/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in index ff9a0e8..ddd3f0b 100644 --- a/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in +++ b/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in @@ -86,15 +86,7 @@ RUN apt-get install -y doxygen graphviz # Client-side `version` command # -RUN apt-get install -y pipx - -USER @DOCKER_FINANCE_USER@ -WORKDIR /home/@DOCKER_FINANCE_USER@ - -RUN pipx install shyaml -RUN echo "export PATH=\"\$PATH:\${HOME}/.local/bin\"" | tee -a ~/.bash_aliases - -USER root +RUN apt-get install -y yq # # Entrypoint diff --git a/client/Dockerfiles/finance/Dockerfile.archlinux.in b/client/Dockerfiles/finance/Dockerfile.archlinux.in index c9c6c98..11d1548 100644 --- a/client/Dockerfiles/finance/Dockerfile.archlinux.in +++ b/client/Dockerfiles/finance/Dockerfile.archlinux.in @@ -47,11 +47,11 @@ RUN pacman -Syu \ vim \ visidata \ xsv \ + yq \ --noconfirm --disable-download-timeout USER @DOCKER_FINANCE_USER@ WORKDIR /home/@DOCKER_FINANCE_USER@ -RUN pipx install shyaml RUN pipx install csvkit # diff --git a/client/Dockerfiles/finance/Dockerfile.ubuntu.in b/client/Dockerfiles/finance/Dockerfile.ubuntu.in index d812961..fab1cd1 100644 --- a/client/Dockerfiles/finance/Dockerfile.ubuntu.in +++ b/client/Dockerfiles/finance/Dockerfile.ubuntu.in @@ -52,6 +52,7 @@ RUN apt-get install -y \ timewarrior \ vim \ visidata \ + yq \ zlib1g-dev RUN apt-get install -y cargo @@ -59,7 +60,6 @@ RUN cargo install xsv --root /usr USER @DOCKER_FINANCE_USER@ WORKDIR /home/@DOCKER_FINANCE_USER@ -RUN pipx install shyaml RUN pipx install csvkit # diff --git a/client/docker-finance.d/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in b/client/docker-finance.d/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in index e2a9746..30a1d34 100644 --- a/client/docker-finance.d/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in +++ b/client/docker-finance.d/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in @@ -24,8 +24,7 @@ USER root RUN apt-get update -y RUN apt-get install -y \ - curl \ - yq + curl USER @DOCKER_FINANCE_USER@ WORKDIR /home/@DOCKER_FINANCE_USER@ diff --git a/client/docker-finance.yaml b/client/docker-finance.yaml index 3baae77..a2bdcab 100644 --- a/client/docker-finance.yaml +++ b/client/docker-finance.yaml @@ -49,13 +49,13 @@ container: - "vim" - "visidata" - "xsv" + - "yq" # For experimental build #- "unuran" commands: - "csvstat --version" - "ghc-pkg --version" - "hledger-flow --version" - - "shyaml --version" fetch: packages: - "composer" @@ -95,6 +95,7 @@ container: - "timewarrior" - "vim" - "visidata" + - "yq" - "zlib1g-dev" # For experimental build #- "libunuran-dev" @@ -102,7 +103,6 @@ container: - "csvstat --version" - "ghc-pkg --version" - "hledger-flow --version" - - "shyaml --version" - "xsv --version" fetch: packages: @@ -128,6 +128,7 @@ container: base: packages: - "composer" + - "yq" commands: linters: packages: diff --git a/client/src/docker/lib/internal/lib_docker.bash b/client/src/docker/lib/internal/lib_docker.bash index d83c307..acc2726 100644 --- a/client/src/docker/lib/internal/lib_docker.bash +++ b/client/src/docker/lib/internal/lib_docker.bash @@ -708,16 +708,19 @@ function lib_docker::__version() ;; esac + local -r _yq=("yq" "-M" "-y" "--indentless" "-e") + local -r _sanitize=("sed" "'s:^- ::'" "|" "grep" "-v" "-E" "'(^null\$|\.\.\.)'") + lib_docker::__run " echo '${_manifest}' \\ - | shyaml keys container.${_platform}.${_image} \\ + | $(echo "${_yq[@]}") \".container.${_platform}.\\\"${_image}\\\" | keys\" | sed \"s:^- ::\" \\ | while read _key; do echo -e \\\ncontainer.${_platform}.${_image}.\${_key}: - echo '${_manifest}' | shyaml get-values container.${_platform}.${_image}.\${_key}.packages 2>/dev/null \\ + echo '${_manifest}' | $(echo "${_yq[@]}") .container.${_platform}.\\\"${_image}\\\".\${_key}.packages | $(echo "${_sanitize[@]}") 2>/dev/null \\ | while read _package; do echo -e \\\t\$($_pkg) done - echo '${_manifest}' | shyaml get-values container.${_platform}.${_image}.\${_key}.commands 2>/dev/null \\ + echo '${_manifest}' | $(echo "${_yq[@]}") .container.${_platform}.\\\"${_image}\\\".\${_key}.commands | $(echo "${_sanitize[@]}") 2>/dev/null \\ | while read _command; do echo -e \\\t\$(\$_command) done diff --git a/container/src/finance/lib/internal/fetch/prices/internal/prices/crypto.php b/container/src/finance/lib/internal/fetch/prices/internal/prices/crypto.php index 19a12cb..3ada71f 100644 --- a/container/src/finance/lib/internal/fetch/prices/internal/prices/crypto.php +++ b/container/src/finance/lib/internal/fetch/prices/internal/prices/crypto.php @@ -50,7 +50,7 @@ namespace docker_finance\prices\internal\prices\crypto $key = $this->get_env()->get_env('API_PRICES_KEY'); $domain = 'api.coingecko.com'; $header = []; - if ($key != 'None') { + if ($key != 'null') { $domain = 'pro-' . $domain; $header = ["x-cg-pro-api-key: $key"]; } @@ -172,7 +172,7 @@ namespace docker_finance\prices\internal\prices\crypto $key = $this->get_env()->get_env('API_PRICES_KEY'); $domain = 'api.mobula.io'; $header = []; - if ($key != 'None') { + if ($key != 'null') { $header = ["Authorization: $key"]; } diff --git a/container/src/finance/lib/internal/lib_fetch.bash b/container/src/finance/lib/internal/lib_fetch.bash index 94d0920..b42c16f 100644 --- a/container/src/finance/lib/internal/lib_fetch.bash +++ b/container/src/finance/lib/internal/lib_fetch.bash @@ -352,6 +352,7 @@ function lib_fetch::__parse_args() fi } +# TODO: complete __fetch() rewrite function lib_fetch::__fetch() { # Supported remote fetch accounts @@ -368,15 +369,17 @@ function lib_fetch::__fetch() "pera-wallet" ) + [ -z "$global_parent_profile" ] && lib_utils::die_fatal + [ -z "$global_child_profile" ] && lib_utils::die_fatal + # TODO: global args should be set in arg parsing and made readonly # Fetch only given accounts if [ ! -z "$global_arg_account" ]; then for _account in "${global_arg_account[@]}"; do - if ! lib_fetch::__parse_yaml "get-value" "account.${_account}" 1 &>/dev/null; then - # Unsupported or unavailable - [ -z "$global_parent_profile" ] && lib_utils::die_fatal - [ -z "$global_child_profile" ] && lib_utils::die_fatal - lib_utils::print_warning "${global_parent_profile}.${global_child_profile}.account.${_account} not found, skipping!" + local _value + _value="$(lib_fetch::__parse_yaml "get-value" "account.${_account}" 2>/dev/null)" + if [[ "$_value" == "null" ]]; then + lib_utils::print_warning "account.${_account} not found, skipping!" else local _accounts+=("$_account") fi @@ -442,39 +445,49 @@ function lib_fetch::__fetch_account() local _members=("key" "passphrase" "secret" "subaccount") for _member in "${_members[@]}"; do + # Get member value - if _value=$(lib_fetch::__parse_yaml "get-value" "account.${_account}.${_member}"); then + local _value + _value=$(lib_fetch::__parse_yaml "get-values" "account.${_account}.${_member}") - # Member value for key - if [[ "$_member" == "key" ]]; then - local _api_key="$_value" - if [[ -z "$_api_key" && $_need_key == true ]]; then - lib_utils::die_fatal "$_account - empty fetch ${_member}!" - fi + if [ $? -eq 0 ]; then + + # Sanitize for 'yq'-specific caveats + local _sanitized + if [[ "$_value" == "null" ]]; then + _sanitized="" + else + # Values may contain 'yq'-interpreted characters (newlines, etc.) + # NOTE: this is needed for at least Coinbase's CDP secret + _sanitized="$(echo "$_value" | sed -e 's:^[ \t]*::' -e '/^$/d' -e "s:^'::" -e "s:'$::")" fi - # Member value for passphrase - if [[ "$_member" == "passphrase" ]]; then - local _api_passphrase="$_value" - if [[ -z "$_api_passphrase" && $_need_passphrase == true ]]; then - lib_utils::die_fatal "$_account - empty fetch ${_member}!" - fi - fi - - # Member value for secret - if [[ "$_member" == "secret" ]]; then - local _api_secret="$_value" - if [[ -z "$_api_secret" && $_need_secret == true ]]; then - lib_utils::die_fatal "$_account - empty fetch ${_member}!" - fi - fi - - # Member value for subaccount (will *always* need subaccount) - if [[ "$_member" == "subaccount" ]]; then - local _api_subaccount="$_value" - [ -z "$_api_subaccount" ] \ - && lib_utils::die_fatal "$_account - empty fetch ${_member}!" - fi + case "$_member" in + key) + local _api_key="$_sanitized" + [[ -z "$_api_key" && $_need_key == true ]] \ + && lib_utils::die_fatal "$_account - empty fetch ${_member}!" + ;; + passphrase) + local _api_passphrase="$_sanitized" + [[ -z "$_api_passphrase" && $_need_passphrase == true ]] \ + && lib_utils::die_fatal "$_account - empty fetch ${_member}!" + ;; + secret) + local _api_secret="$_sanitized" + [[ -z "$_api_secret" && $_need_secret == true ]] \ + && lib_utils::die_fatal "$_account - empty fetch ${_member}!" + ;; + subaccount) + # NOTE: member value for subaccount will *always* need subaccount + local _api_subaccount="$_sanitized" + [ -z "$_api_subaccount" ] \ + && lib_utils::die_fatal "$_account - empty fetch ${_member}!" + ;; + *) + lib_utils::die_fatal "Unsupported member" + ;; + esac else lib_utils::print_warning "$_account is unavailable, skipping!" continue 2 @@ -494,6 +507,7 @@ function lib_fetch::__fetch_account() # (we'll pass entire string to internal fetch impl) # TODO: parse/cut before setting var [ -z "$global_child_profile_flow" ] && lib_utils::die_fatal + local _sub _sub="$(echo $_api_subaccount | cut -d'/' -f1)" local _api_out_dir="${global_child_profile_flow}/import/${global_child_profile}/${_account}/${_sub}/1-in/${global_arg_year}" @@ -508,9 +522,8 @@ function lib_fetch::__fetch_account() # TODO: clarify, add more/better comments - # If subaccount is type 'struct', then subaccount will be list of addresses - # So, create what's needed for internal fetch impl (blockchain:address format) - if [[ $(lib_fetch::__parse_yaml "get-type" "account.${_account}.subaccount") == "struct" ]]; then + # If subaccount is list of addresses, create 'blockchain:address' format (for internal fetch impl) + if echo "$_api_subaccount" | head -n2 | grep ":*$" | tail -n1 | grep -q "^- "; then # Parse out the separate blockchains local _blockchain @@ -520,7 +533,7 @@ function lib_fetch::__fetch_account() _parsed_csv=$( echo "$_blockchain" | while read _key; do local _addresses - _addresses=$(echo "$_api_subaccount" | shyaml get-value "$_key") # TODO: use lib + _addresses=$(echo "$_api_subaccount" | yq -M -y --indentless -e ".${_key}") # TODO: use lib echo "$_addresses" | while read _value; do echo "${_key}:${_value}" | sed -e 's/:- /\//g' -e "s:'::g" done @@ -663,20 +676,24 @@ function lib_fetch::__fetch_price() local _members=("key" "asset") for _member in "${_members[@]}"; do - if _value=$(lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}"); then - if [[ "$_member" == "key" ]]; then - [ -z "$_value" ] \ - && lib_utils::print_warning "${_api} has empty ${_member}" - fi + local _value + _value="$(lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}" 2>/dev/null)" + + case "$_member" in + key) + [[ "$_value" == "null" ]] && lib_utils::print_warning "${_api} has empty ${_member}" + ;; + asset) + if ! lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}" | grep -q "^- "; then + lib_utils::die_fatal "Invalid asset sequence!" + fi + ;; + *) + lib_utils::die_fatal "${_api} has invalid member: ${_member}" + ;; + esac - if [[ "$_member" == "asset" ]]; then - [[ $(lib_fetch::__parse_yaml "get-type" "price.${_api}.${_member}") != "sequence" ]] \ - && lib_utils::die_fatal "Invalid asset sequence!" - fi - else - lib_utils::die_fatal "'${_member}' is unavailable" - fi done # Available API(s) @@ -707,7 +724,7 @@ function lib_fetch::__fetch_price() # Parse out each asset while read _line; do local _assets+=("$_line") - done < <(lib_fetch::__parse_yaml "get-values" "price.${_api}.asset") + done < <(lib_fetch::__parse_yaml "get-values" "price.${_api}.asset" | sed 's:^- ::') else # Get assets from CLI local _assets=("${global_arg_price[*]}") @@ -766,14 +783,34 @@ function lib_fetch::__fetch_price() function lib_fetch::__parse_yaml() { + [[ -z "$1" || -z "$2" ]] && lib_utils::die_fatal local _action="$1" local _append="$2" - [ -z "$_action" ] && lib_utils::die_fatal - [ -z "$_append" ] && lib_utils::die_fatal [ -z "$global_conf_fetch" ] && lib_utils::die_fatal - echo "$(<$global_conf_fetch)" \ - | shyaml -q "$_action" "${global_parent_profile}"."${global_child_profile}"."${_append}" + local -r _raw=".${global_parent_profile}.${global_child_profile}.${_append}" + + # `yq` (kislyuk's) requires quotes around keys/members that contain '-' + local _arg="$_raw" + [[ "$_arg" =~ '-' ]] && _arg="$(sed -e 's:\.:"\.":g' -e 's:"\.:\.:' -e "s: *$:\":" <(echo "$_raw"))" + + # Pseudo-substitution of `shyaml` functionality + local _ifs="$IFS" + IFS=' ' + local -r _cmd=("yq" "-M" "-y" "--indentless" "-e" "$_arg" "$global_conf_fetch") + local -r _grep=("grep" "-v" "^\.\.\.$") + case "$_action" in + get-value) + "${_cmd[@]}" | "${_grep[@]}" -m2 + ;; + get-values) + "${_cmd[@]}" | "${_grep[@]}" + ;; + *) + lib_utils::die_fatal "Unsupported YAML action" + ;; + esac + IFS="$_ifs" } function lib_fetch::__fetch_exec()