Merge pull request #121 into master

bd3dec6 client: lib_docker: version: refactor to use `yq` (Aaron Fiore) c82300b client: manifest: dev-tools: add `yq` (Aaron Fiore) 569db8a client: Dockerfiles: dev-tools: refactor to use `yq` (Aaron Fiore) 29cc6f6 container: fetch: price: refactor empty key type (Aaron Fiore) a3b8890 container: lib_fetch: refactor to use `yq` (Aaron Fiore) 1402284 client: manifest: remove `shyaml` package, add `yq` (Aaron Fiore) 237a4c2 client: Dockerfiles: remove `shyaml` package, add `yq` (Aaron Fiore)
2024-08-17 23:22:17 -07:00
parent ea9ceea06a bd3dec6a94
commit 3b75117796
8 changed files with 107 additions and 75 deletions
--- a/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in
+++ b/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in
@@ -86,15 +86,7 @@ RUN apt-get install -y doxygen graphviz
 # Client-side `version` command
 #

-RUN apt-get install -y pipx
-
-USER @DOCKER_FINANCE_USER@
-WORKDIR /home/@DOCKER_FINANCE_USER@
-
-RUN pipx install shyaml
-RUN echo "export PATH=\"\$PATH:\${HOME}/.local/bin\"" | tee -a ~/.bash_aliases
-
-USER root
+RUN apt-get install -y yq

 #
 # Entrypoint
--- a/client/Dockerfiles/finance/Dockerfile.archlinux.in
+++ b/client/Dockerfiles/finance/Dockerfile.archlinux.in
@@ -47,11 +47,11 @@ RUN pacman -Syu \
  vim \
  visidata \
  xsv \
+  yq \
  --noconfirm --disable-download-timeout

 USER @DOCKER_FINANCE_USER@
 WORKDIR /home/@DOCKER_FINANCE_USER@
-RUN pipx install shyaml
 RUN pipx install csvkit

 #
--- a/client/Dockerfiles/finance/Dockerfile.ubuntu.in
+++ b/client/Dockerfiles/finance/Dockerfile.ubuntu.in
@@ -52,6 +52,7 @@ RUN apt-get install -y \
  timewarrior \
  vim \
  visidata \
+  yq \
  zlib1g-dev

 RUN apt-get install -y cargo
@@ -59,7 +60,6 @@ RUN cargo install xsv --root /usr

 USER @DOCKER_FINANCE_USER@
 WORKDIR /home/@DOCKER_FINANCE_USER@
-RUN pipx install shyaml
 RUN pipx install csvkit

 #
--- a/client/docker-finance.d/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in
+++ b/client/docker-finance.d/client/Dockerfiles/dev-tools/Dockerfile.dev-tools.in
@@ -24,8 +24,7 @@ USER root
 RUN apt-get update -y

 RUN apt-get install -y \
-  curl \
-  yq
+  curl

 USER @DOCKER_FINANCE_USER@
 WORKDIR /home/@DOCKER_FINANCE_USER@
--- a/client/docker-finance.yaml
+++ b/client/docker-finance.yaml
@@ -49,13 +49,13 @@ container:
          - "vim"
          - "visidata"
          - "xsv"
+          - "yq"
          # For experimental build
          #- "unuran"
        commands:
          - "csvstat --version"
          - "ghc-pkg --version"
          - "hledger-flow --version"
-          - "shyaml --version"
      fetch:
        packages:
          - "composer"
@@ -95,6 +95,7 @@ container:
          - "timewarrior"
          - "vim"
          - "visidata"
+          - "yq"
          - "zlib1g-dev"
          # For experimental build
          #- "libunuran-dev"
@@ -102,7 +103,6 @@ container:
          - "csvstat --version"
          - "ghc-pkg --version"
          - "hledger-flow --version"
-          - "shyaml --version"
          - "xsv --version"
      fetch:
        packages:
@@ -128,6 +128,7 @@ container:
      base:
        packages:
          - "composer"
+          - "yq"
        commands:
      linters:
        packages:
--- a/client/src/docker/lib/internal/lib_docker.bash
+++ b/client/src/docker/lib/internal/lib_docker.bash
@@ -708,16 +708,19 @@ function lib_docker::__version()
            ;;
        esac

+        local -r _yq=("yq" "-M" "-y" "--indentless" "-e")
+        local -r _sanitize=("sed" "'s:^- ::'" "|" "grep" "-v" "-E" "'(^null\$|\.\.\.)'")
+
        lib_docker::__run "
          echo '${_manifest}' \\
-            | shyaml keys container.${_platform}.${_image} \\
+            | $(echo "${_yq[@]}") \".container.${_platform}.\\\"${_image}\\\" | keys\" | sed \"s:^- ::\" \\
            | while read _key; do
                echo -e \\\ncontainer.${_platform}.${_image}.\${_key}:
-                echo '${_manifest}' | shyaml get-values container.${_platform}.${_image}.\${_key}.packages 2>/dev/null \\
+                echo '${_manifest}' | $(echo "${_yq[@]}") .container.${_platform}.\\\"${_image}\\\".\${_key}.packages | $(echo "${_sanitize[@]}") 2>/dev/null \\
                | while read _package; do
                    echo -e \\\t\$($_pkg)
                  done
-                echo '${_manifest}' | shyaml get-values container.${_platform}.${_image}.\${_key}.commands 2>/dev/null \\
+                echo '${_manifest}' | $(echo "${_yq[@]}") .container.${_platform}.\\\"${_image}\\\".\${_key}.commands | $(echo "${_sanitize[@]}") 2>/dev/null \\
                | while read _command; do
                    echo -e \\\t\$(\$_command)
                  done
--- a/container/src/finance/lib/internal/fetch/prices/internal/prices/crypto.php
+++ b/container/src/finance/lib/internal/fetch/prices/internal/prices/crypto.php
@@ -50,7 +50,7 @@ namespace docker_finance\prices\internal\prices\crypto
            $key = $this->get_env()->get_env('API_PRICES_KEY');
            $domain = 'api.coingecko.com';
            $header = [];
-            if ($key != 'None') {
+            if ($key != 'null') {
                $domain = 'pro-' . $domain;
                $header = ["x-cg-pro-api-key: $key"];
            }
@@ -172,7 +172,7 @@ namespace docker_finance\prices\internal\prices\crypto
            $key = $this->get_env()->get_env('API_PRICES_KEY');
            $domain = 'api.mobula.io';
            $header = [];
-            if ($key != 'None') {
+            if ($key != 'null') {
                $header = ["Authorization: $key"];
            }

--- a/container/src/finance/lib/internal/lib_fetch.bash
+++ b/container/src/finance/lib/internal/lib_fetch.bash
@@ -352,6 +352,7 @@ function lib_fetch::__parse_args()
  fi
 }

+# TODO: complete __fetch() rewrite
 function lib_fetch::__fetch()
 {
  # Supported remote fetch accounts
@@ -368,15 +369,17 @@ function lib_fetch::__fetch()
    "pera-wallet"
  )

+  [ -z "$global_parent_profile" ] && lib_utils::die_fatal
+  [ -z "$global_child_profile" ] && lib_utils::die_fatal
+
  # TODO: global args should be set in arg parsing and made readonly
  # Fetch only given accounts
  if [ ! -z "$global_arg_account" ]; then
    for _account in "${global_arg_account[@]}"; do
-      if ! lib_fetch::__parse_yaml "get-value" "account.${_account}" 1 &>/dev/null; then
-        # Unsupported or unavailable
-        [ -z "$global_parent_profile" ] && lib_utils::die_fatal
-        [ -z "$global_child_profile" ] && lib_utils::die_fatal
-        lib_utils::print_warning "${global_parent_profile}.${global_child_profile}.account.${_account} not found, skipping!"
+      local _value
+      _value="$(lib_fetch::__parse_yaml "get-value" "account.${_account}" 2>/dev/null)"
+      if [[ "$_value" == "null" ]]; then
+        lib_utils::print_warning "account.${_account} not found, skipping!"
      else
        local _accounts+=("$_account")
      fi
@@ -442,39 +445,49 @@ function lib_fetch::__fetch_account()
    local _members=("key" "passphrase" "secret" "subaccount")

    for _member in "${_members[@]}"; do
+
      # Get member value
-      if _value=$(lib_fetch::__parse_yaml "get-value" "account.${_account}.${_member}"); then
+      local _value
+      _value=$(lib_fetch::__parse_yaml "get-values" "account.${_account}.${_member}")

-        # Member value for key
-        if [[ "$_member" == "key" ]]; then
-          local _api_key="$_value"
-          if [[ -z "$_api_key" && $_need_key == true ]]; then
-            lib_utils::die_fatal "$_account - empty fetch ${_member}!"
-          fi
+      if [ $? -eq 0 ]; then
+
+        # Sanitize for 'yq'-specific caveats
+        local _sanitized
+        if [[ "$_value" == "null" ]]; then
+          _sanitized=""
+        else
+          # Values may contain 'yq'-interpreted characters (newlines, etc.)
+          # NOTE: this is needed for at least Coinbase's CDP secret
+          _sanitized="$(echo "$_value" | sed -e 's:^[ \t]*::' -e '/^$/d' -e "s:^'::" -e "s:'$::")"
        fi

-        # Member value for passphrase
-        if [[ "$_member" == "passphrase" ]]; then
-          local _api_passphrase="$_value"
-          if [[ -z "$_api_passphrase" && $_need_passphrase == true ]]; then
-            lib_utils::die_fatal "$_account - empty fetch ${_member}!"
-          fi
-        fi
-
-        # Member value for secret
-        if [[ "$_member" == "secret" ]]; then
-          local _api_secret="$_value"
-          if [[ -z "$_api_secret" && $_need_secret == true ]]; then
-            lib_utils::die_fatal "$_account - empty fetch ${_member}!"
-          fi
-        fi
-
-        # Member value for subaccount (will *always* need subaccount)
-        if [[ "$_member" == "subaccount" ]]; then
-          local _api_subaccount="$_value"
-          [ -z "$_api_subaccount" ] \
-            && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
-        fi
+        case "$_member" in
+          key)
+            local _api_key="$_sanitized"
+            [[ -z "$_api_key" && $_need_key == true ]] \
+              && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
+            ;;
+          passphrase)
+            local _api_passphrase="$_sanitized"
+            [[ -z "$_api_passphrase" && $_need_passphrase == true ]] \
+              && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
+            ;;
+          secret)
+            local _api_secret="$_sanitized"
+            [[ -z "$_api_secret" && $_need_secret == true ]] \
+              && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
+            ;;
+          subaccount)
+            # NOTE: member value for subaccount will *always* need subaccount
+            local _api_subaccount="$_sanitized"
+            [ -z "$_api_subaccount" ] \
+              && lib_utils::die_fatal "$_account - empty fetch ${_member}!"
+            ;;
+          *)
+            lib_utils::die_fatal "Unsupported member"
+            ;;
+        esac
      else
        lib_utils::print_warning "$_account is unavailable, skipping!"
        continue 2
@@ -494,6 +507,7 @@ function lib_fetch::__fetch_account()
    #   (we'll pass entire string to internal fetch impl)
    # TODO: parse/cut before setting var
    [ -z "$global_child_profile_flow" ] && lib_utils::die_fatal
+
    local _sub
    _sub="$(echo $_api_subaccount | cut -d'/' -f1)"
    local _api_out_dir="${global_child_profile_flow}/import/${global_child_profile}/${_account}/${_sub}/1-in/${global_arg_year}"
@@ -508,9 +522,8 @@ function lib_fetch::__fetch_account()

    # TODO: clarify, add more/better comments

-    # If subaccount is type 'struct', then subaccount will be list of addresses
-    # So, create what's needed for internal fetch impl (blockchain:address format)
-    if [[ $(lib_fetch::__parse_yaml "get-type" "account.${_account}.subaccount") == "struct" ]]; then
+    # If subaccount is list of addresses, create 'blockchain:address' format (for internal fetch impl)
+    if echo "$_api_subaccount" | head -n2 | grep ":*$" | tail -n1 | grep -q "^- "; then

      # Parse out the separate blockchains
      local _blockchain
@@ -520,7 +533,7 @@ function lib_fetch::__fetch_account()
      _parsed_csv=$(
        echo "$_blockchain" | while read _key; do
          local _addresses
-          _addresses=$(echo "$_api_subaccount" | shyaml get-value "$_key") # TODO: use lib
+          _addresses=$(echo "$_api_subaccount" | yq -M -y --indentless -e ".${_key}") # TODO: use lib
          echo "$_addresses" | while read _value; do
            echo "${_key}:${_value}" | sed -e 's/:- /\//g' -e "s:'::g"
          done
@@ -663,20 +676,24 @@ function lib_fetch::__fetch_price()

    local _members=("key" "asset")
    for _member in "${_members[@]}"; do
-      if _value=$(lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}"); then

-        if [[ "$_member" == "key" ]]; then
-          [ -z "$_value" ] \
-            && lib_utils::print_warning "${_api} has empty ${_member}"
-        fi
+      local _value
+      _value="$(lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}" 2>/dev/null)"
+
+      case "$_member" in
+        key)
+          [[ "$_value" == "null" ]] && lib_utils::print_warning "${_api} has empty ${_member}"
+          ;;
+        asset)
+          if ! lib_fetch::__parse_yaml "get-value" "price.${_api}.${_member}" | grep -q "^- "; then
+            lib_utils::die_fatal "Invalid asset sequence!"
+          fi
+          ;;
+        *)
+          lib_utils::die_fatal "${_api} has invalid member: ${_member}"
+          ;;
+      esac

-        if [[ "$_member" == "asset" ]]; then
-          [[ $(lib_fetch::__parse_yaml "get-type" "price.${_api}.${_member}") != "sequence" ]] \
-            && lib_utils::die_fatal "Invalid asset sequence!"
-        fi
-      else
-        lib_utils::die_fatal "'${_member}' is unavailable"
-      fi
    done

    # Available API(s)
@@ -707,7 +724,7 @@ function lib_fetch::__fetch_price()
      # Parse out each asset
      while read _line; do
        local _assets+=("$_line")
-      done < <(lib_fetch::__parse_yaml "get-values" "price.${_api}.asset")
+      done < <(lib_fetch::__parse_yaml "get-values" "price.${_api}.asset" | sed 's:^- ::')
    else
      # Get assets from CLI
      local _assets=("${global_arg_price[*]}")
@@ -766,14 +783,34 @@ function lib_fetch::__fetch_price()

 function lib_fetch::__parse_yaml()
 {
+  [[ -z "$1" || -z "$2" ]] && lib_utils::die_fatal
  local _action="$1"
  local _append="$2"
-  [ -z "$_action" ] && lib_utils::die_fatal
-  [ -z "$_append" ] && lib_utils::die_fatal

  [ -z "$global_conf_fetch" ] && lib_utils::die_fatal
-  echo "$(<$global_conf_fetch)" \
-    | shyaml -q "$_action" "${global_parent_profile}"."${global_child_profile}"."${_append}"
+  local -r _raw=".${global_parent_profile}.${global_child_profile}.${_append}"
+
+  # `yq` (kislyuk's) requires quotes around keys/members that contain '-'
+  local _arg="$_raw"
+  [[ "$_arg" =~ '-' ]] && _arg="$(sed -e 's:\.:"\.":g' -e 's:"\.:\.:' -e "s: *$:\":" <(echo "$_raw"))"
+
+  # Pseudo-substitution of `shyaml` functionality
+  local _ifs="$IFS"
+  IFS=' '
+  local -r _cmd=("yq" "-M" "-y" "--indentless" "-e" "$_arg" "$global_conf_fetch")
+  local -r _grep=("grep" "-v" "^\.\.\.$")
+  case "$_action" in
+    get-value)
+      "${_cmd[@]}" | "${_grep[@]}" -m2
+      ;;
+    get-values)
+      "${_cmd[@]}" | "${_grep[@]}"
+      ;;
+    *)
+      lib_utils::die_fatal "Unsupported YAML action"
+      ;;
+  esac
+  IFS="$_ifs"
 }

 function lib_fetch::__fetch_exec()