From 52026cace17c92f920cdf9d43175195e929cc8f5 Mon Sep 17 00:00:00 2001 From: Aaron Fiore Date: Fri, 14 Jun 2024 16:26:56 -0700 Subject: [PATCH] php: fetch: prices: new functionality / refactor - Add support for multiple aggregator APIs - Refactor `prices` API implementation - Update documentation --- client/Doxygen/docker-finance.dox | 4 + .../src/finance/lib/internal/fetch/fetch.php | 8 +- .../lib/internal/fetch/prices/fetch.php | 15 +- .../internal/fetch/prices/internal/base.php | 283 +++++++++++++++++- .../prices/internal/prices/coingecko.php | 261 ++++------------ 5 files changed, 349 insertions(+), 222 deletions(-) diff --git a/client/Doxygen/docker-finance.dox b/client/Doxygen/docker-finance.dox index a09c67b..0fe357b 100644 --- a/client/Doxygen/docker-finance.dox +++ b/client/Doxygen/docker-finance.dox @@ -108,6 +108,10 @@ //! \brief Group for supported exchanges //! \since docker-finance 1.0.0 +//! \defgroup php_prices docker-finance PHP prices +//! \brief Group for supported price aggregators +//! \since docker-finance 1.0.0 + //! \defgroup php_utils docker-finance PHP utilities //! \brief Group for internal utility code //! \since docker-finance 1.0.0 diff --git a/container/src/finance/lib/internal/fetch/fetch.php b/container/src/finance/lib/internal/fetch/fetch.php index dbcef4e..5f17011 100644 --- a/container/src/finance/lib/internal/fetch/fetch.php +++ b/container/src/finance/lib/internal/fetch/fetch.php @@ -63,7 +63,9 @@ namespace docker_finance $env->set_env('API_OUT_DIR', getenv('API_OUT_DIR') . '/'); // API output path (account's 1-in dir) // Prices - $env->set_env('API_PRICES_PATH', getenv('API_PRICES_PATH')); // Master file + $env->set_env('API_PRICES_PATH', getenv('API_PRICES_PATH')); // Master price file + $env->set_env('API_PRICES_API', getenv('API_PRICES_API')); // Price API to use + $env->set_env('API_PRICES_KEY', getenv('API_PRICES_KEY')); // Price API key $env->set_env('API_PRICES_SYMBOLS', getenv('API_PRICES_SYMBOLS')); // User-provided symbols // Exchanges @@ -104,11 +106,11 @@ namespace docker_finance } break; case 'prices': - switch ($env->get_env('API_FETCH_SUBTYPE')) { + switch ($subtype) { case 'crypto': + // TODO: `case 'legacy'` (stocks and bonds and ETFs, oh my!) $api = new prices\Fetch($env); break; - // TODO: stocks and bonds and ETFs, oh my! default: utils\CLI::throw_fatal("unsupported subtype '$subtype' for interal API"); break; diff --git a/container/src/finance/lib/internal/fetch/prices/fetch.php b/container/src/finance/lib/internal/fetch/prices/fetch.php index 462dee7..a35c4c5 100644 --- a/container/src/finance/lib/internal/fetch/prices/fetch.php +++ b/container/src/finance/lib/internal/fetch/prices/fetch.php @@ -40,7 +40,7 @@ namespace docker_finance\prices */ final class Fetch extends API { - private internal\prices\Crypto $api; //!< Internal API + private mixed $api; //!< Internal API public function __construct(utils\Env $env) { @@ -50,14 +50,17 @@ namespace docker_finance\prices //! @brief Fetch executor public function fetch(): void { - $subtype = $this->get_env()->get_env('API_FETCH_SUBTYPE'); - switch ($subtype) { - case 'crypto': - $this->api = new internal\prices\Crypto($this->get_env()); + $upstream = $this->get_env()->get_env('API_PRICES_API'); + switch ($upstream) { + case 'coingecko': + $this->api = new internal\prices\CoinGecko($this->get_env()); + break; + case 'mobula': + $this->api = new internal\prices\Mobula($this->get_env()); break; default: utils\CLI::throw_fatal( - "unsupported subtype '$subtype' for interal API" + "unsupported upstream API '{$upstream}' for interal API" ); break; } diff --git a/container/src/finance/lib/internal/fetch/prices/internal/base.php b/container/src/finance/lib/internal/fetch/prices/internal/base.php index 1f35ace..fee38a3 100644 --- a/container/src/finance/lib/internal/fetch/prices/internal/base.php +++ b/container/src/finance/lib/internal/fetch/prices/internal/base.php @@ -31,10 +31,49 @@ namespace docker_finance\prices\internal use docker_finance\utils as utils; /** - * @brief Base internal API + * @brief Common implementaion interface + * @details Unified with top-level API interace * @since docker-finance 1.0.0 */ - abstract class API + interface ImplInterface + { + /** + * @brief Get network response data + * @param string $id ID of given symbol ('bitcoin' in 'bitcoin/BTC') + * @param string $timestamp Given year(s) to fetch + * @return array Array of [N]([timestamp][price]) for given year(s) + */ + public function getter(string $id, string $timestamp): array; + + /** + * @brief Prepare price data for given symbols + * @param string $symbols Symbols in 'asset/ticker,...' format ('bitcoin/BTC,ethereum/ETH') + * @return array, array> Prices for all given symbols + */ + public function reader(string $symbols): array; + + /** + * @brief Write price data to master prices journal + * @param array> $data Prepared data for master prices journal + * @param string $path Full path to journal + * @note External implementation *MUST* append and sort to every applicable year + * @warning Clobbers external master price journal + * @todo Enforce array for stack + */ + public function writer(mixed $data, string $path): void; + + /** + * @brief Caller for entire fetch process + * @details Handler for calling getter, reader and writer + */ + public function fetcher(): void; + } + + /** + * @brief Common implementation + * @since docker-finance 1.0.0 + */ + abstract class Impl implements ImplInterface { private utils\Env $env; //!< Environment @@ -48,11 +87,249 @@ namespace docker_finance\prices\internal return $this->env; } - abstract public function fetch(): void; + /** + * @brief Parse given string of 'id/symbol' + * @param string $symbols Expected format: bitcoin/BTC,ethereum/ETH,litecoin/LTC,etc. + * @details + * + * Caveats: + * + * 1. There are multiple reasons why asset must be passed with symbol + * instead of symbol alone: + * + * a. CoinGecko uses same symbol for multiple ID's: + * + * ltc = binance-peg-litecoin + * ltc = litecoin + * eth = ethereum + * eth = ethereum-wormhole + * + * So, pass ID instead of symbol! + * + * b. Mobula support will require an asset name along with symbol + * + * 2. Ticker-symbol comes *AFTER* ID because hledger's prices are: + * + * a. *CASE SENSITIVE* + * + * b *WILL NOT UNDERSTAND THE DIFFERENCE BETWEEN (for example): + * aGUSD and AGUSD* (CoinGecko will return lowercase symbol) + * + * @return array Array of symbols + */ + protected function parse_symbols(string $symbols): array + { + utils\CLI::print_debug($symbols); + + if (!str_contains($this->get_env()->get_env('API_PRICES_SYMBOLS'), '/')) { + utils\CLI::throw_fatal("malformed symbols format"); + } + + $list = explode(',', $symbols); + $parsed = []; + + foreach ($list as $coin) { + list($key, $value) = explode('/', $coin); + $parsed[$key] = $value; + } + return $parsed; + } + + /** + * @brief Request's common implementation + * @param string $url REST API URL + * @return mixed Response data + */ + protected function request_impl(string $url): mixed + { + $headers = array( + 'Accept: application/json', + 'Content-Type: application/json', + ); + + $ch = curl_init($url); + + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HEADER, false); + + $response = curl_exec($ch); + + $info = curl_getinfo($ch); + utils\CLI::print_debug($info); + + curl_close($ch); + + if ($response === false) { + utils\CLI::throw_fatal("cURL null response"); + } + + return $response; + } + + //! @brief Impl-specific REST API request + abstract protected function request(string $id, string $timestamp): mixed; + + /** + * @brief Make impl-specific timestamp requirement + * @param string $year Given year + */ + abstract protected function make_timestamp(string $year): mixed; + + /** + * @brief Parse fetched prices by symbol + * @param string $symbol Given symbol associated with ID + * @param array $prices Array of [N]([timestamp][price])for given year(s) + * @return array Prices for all given symbols + */ + abstract protected function parse_prices(string $symbol, array $prices): array; + + /** + * @brief Create data for master price journal file + * @param string $symbol Given symbol associated with ID + * @param array $prices Array of [N]([timestamp][price])for given year(s) + * @return array Master price journal file data + */ + protected function make_master(string $symbol, array $prices): array + { + $stack = []; // Final journal entries + $average = 0; // Purely for printing + + foreach ($prices as $date => $price) { + + // Price journal entry line + $line = 'P ' . $date . ' ' . $symbol . ' ' . sprintf('%.8f', $price) . "\n"; + array_push($stack, $line); + + // Always push a placeholder $/USD for hledger calculations. + // This is so there aren't separate output lines from the + // `--value` calculated and current $/USD holdings. + $line = 'P ' . $date . ' ' . '$' . ' ' . '1' . "\n"; + array_push($stack, $line); + + $line = 'P ' . $date . ' ' . 'USD' . ' ' . '1' . "\n"; + array_push($stack, $line); + + // + // HACKS to get USD amount of unsupported upstream coins + // + + if ($symbol == 'AAVE') { + $line = 'P ' . $date . ' ' . 'stkAAVE' . ' ' . sprintf('%.8f', $price) . "\n"; + array_push($stack, $line); + } + + // Hack for array('paxos-standard'=>'USDP') + if ($symbol == 'PAX') { + $line = 'P ' . $date . ' ' . 'USDP' . ' ' . sprintf('%.8f', $price) . "\n"; + array_push($stack, $line); + } + + // CGLD was changed to CELO at some point + if ($symbol == 'CGLD') { + $line = 'P ' . $date . ' ' . 'CELO' . ' ' . sprintf('%.8f', $price) . "\n"; + array_push($stack, $line); + } + + // Clobber into most-recent daily average of given year + $average = $price; + } + + // Print symbol and most recent price parsed + utils\CLI::print_custom(" \e[32m│\e[0m\n"); + utils\CLI::print_custom(" \e[32m├─\e[34m\e[1;3m $symbol\e[0m\n"); + utils\CLI::print_custom(" \e[32m│ └─\e[37;2m " . $average . "\e[0m\n"); + + return $stack; + } + + public function getter(string $id, string $timestamp): array + { + $response = []; + $timer = 60; // seconds + $success = false; + + while (!$success) { + try { + $response = $this->request($id, $timestamp); + $success = true; // Should throw before this is assigned, alla C++ + } catch (\Throwable $e) { + $code = $e->getCode(); + if ($code == 429) { + $code = "429 (server rate-limiting)"; + } + utils\CLI::print_warning( + "server sent error '" . $code . "' for '$id' ! Trying again in $timer seconds" + ); + $i = 1; + $j = 1; + while ($i <= $timer) { + if ($j == 10) { + utils\CLI::print_custom("\e[33;1m+\e[0m"); + $j = 0; + } else { + utils\CLI::print_custom("\e[33m.\e[0m"); + } + sleep(1); + $i++; + $j++; + } + utils\CLI::print_custom("\n"); + } + } + + utils\CLI::print_debug($response); + return $response; + } + + public function reader(string $symbols): array + { + // Fetched prices for given symbols + $stack = []; + + // Timestamp based on given year + $timestamp = $this->make_timestamp($this->get_env()->get_env('API_FETCH_YEAR')); + + utils\CLI::print_normal(" ─ Symbols"); + + foreach ($this->parse_symbols($symbols) as $id => $symbol) { + $parsed = $this->parse_prices( + $symbol, + $this->getter($id, $timestamp) + ); + array_push($stack, $this->make_master($symbol, $parsed)); + } + + utils\CLI::print_custom(" \e[32m│\e[0m\n"); + + return $stack; + } + + public function writer(mixed $data, string $path): void + { + // Cohesive array of all fetched symbols + $stack = []; + + // Each element is an array of symbol prices + foreach($data as $symbol) { + foreach($symbol as $price) { + array_push($stack, $price); + } + } + + file_put_contents($path, array_unique($stack), LOCK_EX); + } + + public function fetcher(): void + { + $master = $this->reader($this->get_env()->get_env('API_PRICES_SYMBOLS')); + $this->writer($master, $this->get_env()->get_env('API_PRICES_PATH')); + } } } // namespace docker_finance\prices\internal //! @since docker-finance 1.0.0 + namespace docker_finance\prices { require_once('utils/utils.php'); diff --git a/container/src/finance/lib/internal/fetch/prices/internal/prices/coingecko.php b/container/src/finance/lib/internal/fetch/prices/internal/prices/coingecko.php index 38f43ae..b3c15a3 100644 --- a/container/src/finance/lib/internal/fetch/prices/internal/prices/coingecko.php +++ b/container/src/finance/lib/internal/fetch/prices/internal/prices/coingecko.php @@ -24,19 +24,20 @@ */ //! @since docker-finance 1.0.0 -namespace docker_finance\prices\internal\prices +namespace docker_finance\prices\internal\prices\coingecko { require_once('php/vendor/autoload.php'); //!< CoinGecko require_once('prices/internal/base.php'); require_once('utils/utils.php'); + use docker_finance\prices\internal as internal; use docker_finance\utils as utils; /** - * @brief CoinGecko cryptocurrency API + * @brief CoinGecko price aggregator API * @since docker-finance 1.0.0 */ - abstract class CoinGecko extends \docker_finance\prices\internal\API + final class CoinGecko extends internal\Impl { private \Codenixsv\CoinGeckoApi\CoinGeckoClient $api; //!< CoinGecko @@ -48,55 +49,35 @@ namespace docker_finance\prices\internal\prices } /** - * @brief Parse given string of 'id/symbol' - * @param string $symbols Expected format: bitcoin/BTC,ethereum/ETH,litecoin/LTC,etc. - * - * @details - * - * CoinGecko caveats: - * - * 1. The reason why ID must be entered and not symbol is because - * coingecko uses same symbol for multiple ID's: - * - * ltc = binance-peg-litecoin - * ltc = litecoin - * eth = ethereum - * eth = ethereum-wormhole - * - * So, pass ID instead of symbol! - * - * 2. Put ticker-symbol *AFTER* ID because hledger's prices are: - * - * - *CASE SENSITIVE* - * - * - *WILL NOT UNDERSTAND THE DIFFERENCE BETWEEN (for example): - * aGUSD and AGUSD* (CoinGecko will return lowercase symbol) - * - * @return array Array of symbols + * @brief REST API request generator + * @param string $id Symbol's ID to request + * @param string $timestamp Timestamp to request + * @return mixed REST API response data */ - private function parse_symbols(string $symbols): array + protected function request(string $id, string $timestamp): mixed { - if (!str_contains($this->get_env()->get_env('API_PRICES_SYMBOLS'), '/')) { - utils\CLI::throw_fatal("malformed symbols format"); - } - - $list = explode(',', $symbols); - $parsed = []; - - foreach ($list as $coin) { - list($key, $value) = explode('/', $coin); - $parsed[$key] = $value; - } - return $parsed; + // TODO(afiore): use request_impl() after removing CoinGeckoClient + $response = $this->api->coins()->getMarketChart($id, 'usd', $timestamp); + return $response['prices']; } /** - * @brief Parse prices + * @brief Parse given prices for given symbol * @param string $symbol Given symbol associated with CoinGecko's ID * @param array $prices Array of [N]([timestamp][price])for given year(s) + * @details + * + * Parses historical market data include price, market cap + * and 24h volume (granularity auto) + * + * - Data granularity is automatic (cannot be adjusted) + * - 1 day from current time = 5 minute interval data + * - 1 - 90 days from current time = hourly data + * above 90 days from current time = daily data (00:00 UTC) + * * @return array Prices for all given symbols */ - private function parse_prices(string $symbol, array $prices): array + protected function parse_prices(string $symbol, array $prices): array { $total = 0; $prev_date = ""; @@ -183,66 +164,15 @@ namespace docker_finance\prices\internal\prices $prev_date = $date; } - // - // Create final stack of journal entries - // - - $stack = []; // Final journal entries - $average = 0; // Purely for printing - - foreach ($prices_stack as $date => $price) { - - // Price journal entry line - $line = 'P ' . $date . ' ' . $symbol . ' ' . sprintf('%.8f', $price) . "\n"; - array_push($stack, $line); - - // Always push a placeholder $/USD for hledger calculations. - // This is so there aren't separate output lines from the - // `--value` calculated and current $/USD holdings. - $line = 'P ' . $date . ' ' . '$' . ' ' . '1' . "\n"; - array_push($stack, $line); - - $line = 'P ' . $date . ' ' . 'USD' . ' ' . '1' . "\n"; - array_push($stack, $line); - - // - // HACKS to get USD amount of unsupported upstream coins - // - - if ($symbol == 'AAVE') { - $line = 'P ' . $date . ' ' . 'stkAAVE' . ' ' . sprintf('%.8f', $price) . "\n"; - array_push($stack, $line); - } - - // Hack for array('paxos-standard'=>'USDP') - if ($symbol == 'PAX') { - $line = 'P ' . $date . ' ' . 'USDP' . ' ' . sprintf('%.8f', $price) . "\n"; - array_push($stack, $line); - } - - // CGLD was changed to CELO at some point, let's add CELO as well - if ($symbol == 'CGLD') { - $line = 'P ' . $date . ' ' . 'CELO' . ' ' . sprintf('%.8f', $price) . "\n"; - array_push($stack, $line); - } - - // Clobber into most-recent daily average of given year - $average = $price; - } - - // Print symbol and most recent price parsed - utils\CLI::print_custom(" \e[32m│\e[0m\n"); - utils\CLI::print_custom(" \e[32m├─\e[34m\e[1;3m $symbol\e[0m\n"); - utils\CLI::print_custom(" \e[32m│ └─\e[37;2m " . $average . "\e[0m\n"); - - return $stack; + return $prices_stack; } /** - * @note satisfies date for CoinGecko API + * @brief Make CoinGecko timestamp used in request * @param string $year Given year + * @return mixed Number of days since given year */ - private function get_days(string $year): mixed + protected function make_timestamp(string $year): mixed { // Number of days back to beginning of given year if ($year != 'all') { @@ -269,126 +199,37 @@ namespace docker_finance\prices\internal\prices // All possible dates return 'max'; } + } +} // namespace docker_finance\prices\internal\prices\coingecko - /** - * @brief Get prices - * @param string $id CoinGecko's ID of given symbol - * @param string $year Given year(s) to fetch (year or 'max') - * @details - * - * Gets historical market data include price, market cap - * and 24h volume (granularity auto) - * - * - Data granularity is automatic (cannot be adjusted) - * - 1 day from current time = 5 minute interval data - * - 1 - 90 days from current time = hourly data - * above 90 days from current time = daily data (00:00 UTC) - * - * @return array Array of [N]([timestamp][price]) for given year(s) - */ - private function get_prices(string $id, string $year): array +//! @since docker-finance 1.0.0 + +namespace docker_finance\prices\internal\prices +{ + require_once('prices/internal/base.php'); + require_once('utils/utils.php'); + + use docker_finance\utils as utils; + + /** + * @brief Facade for CoinGecko implementation + * @ingroup php_prices + * @since docker-finance 1.0.0 + */ + final class CoinGecko extends \docker_finance\prices\API + { + private coingecko\CoinGecko $api; //!< Internal API + + public function __construct(utils\Env $env) { - $chart = []; // fetched chart - $timer = 60; // seconds - $success = false; - - while (!$success) { - try { - $chart = $this->api->coins()->getMarketChart($id, 'usd', $year); - $success = true; // Should throw before this is assigned, alla C++ - } catch (\Throwable $e) { - $code = $e->getCode(); - if ($code == 429) { - $code = "429 (server rate-limiting)"; - } - utils\CLI::print_warning( - "server sent error '" . $code . "' for '$id' ! Trying again in $timer seconds" - ); - $i = 1; - $j = 1; - while ($i <= $timer) { - if ($j == 10) { - utils\CLI::print_custom("\e[33;1m+\e[0m"); - $j = 0; - } else { - utils\CLI::print_custom("\e[33m.\e[0m"); - } - sleep(1); - $i++; - $j++; - } - utils\CLI::print_custom("\n"); - } - } - - $stack = $chart['prices']; - utils\CLI::print_debug($stack); - - return $stack; - } - - /** - * @brief Get given symbols - * @return array, array> Prices for all given symbols - */ - protected function get(string $symbols): array - { - // Fetched prices for given symbols - $stack = []; - - utils\CLI::print_normal(" ─ Symbols"); - - foreach ($this->parse_symbols($symbols) as $id => $symbol) { - array_push($stack, $this->parse_prices( - $symbol, - $this->get_prices( - $id, - $this->get_days($this->get_env()->get_env('API_FETCH_YEAR')) - ) - )); - } - - utils\CLI::print_custom(" \e[32m│\e[0m\n"); - - return $stack; - } - - /** - * @brief Write prices master file - * @param array> $data Data to write - * @param string $path Path to file - * @note External impl *MUST* append and sort to every applicable year - * @warning Clobbers external master price file - */ - protected function write(array $data, string $path): void - { - // Cohesive array of all fetched symbols - $stack = []; - - // Each element is an array of symbol prices - foreach($data as $symbol) { - foreach($symbol as $price) { - array_push($stack, $price); - } - } - - file_put_contents($path, array_unique($stack), LOCK_EX); + $this->api = new coingecko\CoinGecko($env); } public function fetch(): void { - $prices = $this->get($this->get_env()->get_env('API_PRICES_SYMBOLS')); - $this->write($prices, $this->get_env()->get_env('API_PRICES_PATH')); + $this->api->fetcher(); } } - - /** - * @brief Cryptocurrency prices API - * @since docker-finance 1.0.0 - */ - final class Crypto extends CoinGecko - { - } } // namespace docker_finance\prices\internal\prices # vim: sw=4 sts=4 si ai et