AaronFiore/docker-finance
1
0
Fork 0
forked from EvergreenCrypto/docker-finance
Code Activity

php: fetch: prices: new functionality / refactor

Browse Source 
- Add support for multiple aggregator APIs
- Refactor `prices` API implementation
- Update documentation
This commit is contained in:
Aaron Fiore
2024-06-14 16:26:56 -07:00
parent e9e8b84bd8
commit 52026cace1
5 changed files with 349 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
container/src/finance/lib/internal/fetch/fetch.php
View File

@@ -63,7 +63,9 @@ namespace docker_finance
$env->set_env('API_OUT_DIR', getenv('API_OUT_DIR') . '/'); // API output path (account's 1-in dir)
// Prices
$env->set_env('API_PRICES_PATH', getenv('API_PRICES_PATH')); // Master file
$env->set_env('API_PRICES_PATH', getenv('API_PRICES_PATH')); // Master price file
$env->set_env('API_PRICES_API', getenv('API_PRICES_API')); // Price API to use
$env->set_env('API_PRICES_KEY', getenv('API_PRICES_KEY')); // Price API key
$env->set_env('API_PRICES_SYMBOLS', getenv('API_PRICES_SYMBOLS')); // User-provided symbols
// Exchanges
@@ -104,11 +106,11 @@ namespace docker_finance
}
break;
case 'prices':
switch ($env->get_env('API_FETCH_SUBTYPE')) {
switch ($subtype) {
case 'crypto':
// TODO: `case 'legacy'` (stocks and bonds and ETFs, oh my!)
$api = new prices\Fetch($env);
break;
// TODO: stocks and bonds and ETFs, oh my!
default:
utils\CLI::throw_fatal("unsupported subtype '$subtype' for interal API");
break;

15
container/src/finance/lib/internal/fetch/prices/fetch.php
View File

@@ -40,7 +40,7 @@ namespace docker_finance\prices
*/
final class Fetch extends API
{
private internal\prices\Crypto $api; //!< Internal API
private mixed $api; //!< Internal API
public function __construct(utils\Env $env)
{
@@ -50,14 +50,17 @@ namespace docker_finance\prices
//! @brief Fetch executor
public function fetch(): void
{
$subtype = $this->get_env()->get_env('API_FETCH_SUBTYPE');
switch ($subtype) {
case 'crypto':
$this->api = new internal\prices\Crypto($this->get_env());
$upstream = $this->get_env()->get_env('API_PRICES_API');
switch ($upstream) {
case 'coingecko':
$this->api = new internal\prices\CoinGecko($this->get_env());
break;
case 'mobula':
$this->api = new internal\prices\Mobula($this->get_env());
break;
default:
utils\CLI::throw_fatal(
"unsupported subtype '$subtype' for interal API"
"unsupported upstream API '{$upstream}' for interal API"
);
break;
}

283
container/src/finance/lib/internal/fetch/prices/internal/base.php
View File

@@ -31,10 +31,49 @@ namespace docker_finance\prices\internal
use docker_finance\utils as utils;
/**
* @brief Base internal API
* @brief Common implementaion interface
* @details Unified with top-level API interace
* @since docker-finance 1.0.0
*/
abstract class API
interface ImplInterface
{
/**
* @brief Get network response data
* @param string $id ID of given symbol ('bitcoin' in 'bitcoin/BTC')
* @param string $timestamp Given year(s) to fetch
* @return array<mixed> Array of [N]([timestamp][price]) for given year(s)
*/
public function getter(string $id, string $timestamp): array;
/**
* @brief Prepare price data for given symbols
* @param string $symbols Symbols in 'asset/ticker,...' format ('bitcoin/BTC,ethereum/ETH')
* @return array<int<0, max>, array<string>> Prices for all given symbols
*/
public function reader(string $symbols): array;
/**
* @brief Write price data to master prices journal
* @param array<array<string>> $data Prepared data for master prices journal
* @param string $path Full path to journal
* @note External implementation *MUST* append and sort to every applicable year
* @warning Clobbers external master price journal
* @todo Enforce array for stack
*/
public function writer(mixed $data, string $path): void;
/**
* @brief Caller for entire fetch process
* @details Handler for calling getter, reader and writer
*/
public function fetcher(): void;
}
/**
* @brief Common implementation
* @since docker-finance 1.0.0
*/
abstract class Impl implements ImplInterface
{
private utils\Env $env; //!< Environment
@@ -48,11 +87,249 @@ namespace docker_finance\prices\internal
return $this->env;
}
abstract public function fetch(): void;
/**
* @brief Parse given string of 'id/symbol'
* @param string $symbols Expected format: bitcoin/BTC,ethereum/ETH,litecoin/LTC,etc.
* @details
*
* Caveats:
*
* 1. There are multiple reasons why asset must be passed with symbol
* instead of symbol alone:
*
* a. CoinGecko uses same symbol for multiple ID's:
*
* ltc = binance-peg-litecoin
* ltc = litecoin
* eth = ethereum
* eth = ethereum-wormhole
*
* So, pass ID instead of symbol!
*
* b. Mobula support will require an asset name along with symbol
*
* 2. Ticker-symbol comes *AFTER* ID because hledger's prices are:
*
* a. *CASE SENSITIVE*
*
* b *WILL NOT UNDERSTAND THE DIFFERENCE BETWEEN (for example):
* aGUSD and AGUSD* (CoinGecko will return lowercase symbol)
*
* @return array<string> Array of symbols
*/
protected function parse_symbols(string $symbols): array
{
utils\CLI::print_debug($symbols);
if (!str_contains($this->get_env()->get_env('API_PRICES_SYMBOLS'), '/')) {
utils\CLI::throw_fatal("malformed symbols format");
}
$list = explode(',', $symbols);
$parsed = [];
foreach ($list as $coin) {
list($key, $value) = explode('/', $coin);
$parsed[$key] = $value;
}
return $parsed;
}
/**
* @brief Request's common implementation
* @param string $url REST API URL
* @return mixed Response data
*/
protected function request_impl(string $url): mixed
{
$headers = array(
'Accept: application/json',
'Content-Type: application/json',
);
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HEADER, false);
$response = curl_exec($ch);
$info = curl_getinfo($ch);
utils\CLI::print_debug($info);
curl_close($ch);
if ($response === false) {
utils\CLI::throw_fatal("cURL null response");
}
return $response;
}
//! @brief Impl-specific REST API request
abstract protected function request(string $id, string $timestamp): mixed;
/**
* @brief Make impl-specific timestamp requirement
* @param string $year Given year
*/
abstract protected function make_timestamp(string $year): mixed;
/**
* @brief Parse fetched prices by symbol
* @param string $symbol Given symbol associated with ID
* @param array<mixed> $prices Array of [N]([timestamp][price])for given year(s)
* @return array<string> Prices for all given symbols
*/
abstract protected function parse_prices(string $symbol, array $prices): array;
/**
* @brief Create data for master price journal file
* @param string $symbol Given symbol associated with ID
* @param array<mixed> $prices Array of [N]([timestamp][price])for given year(s)
* @return array<string> Master price journal file data
*/
protected function make_master(string $symbol, array $prices): array
{
$stack = []; // Final journal entries
$average = 0; // Purely for printing
foreach ($prices as $date => $price) {
// Price journal entry line
$line = 'P ' . $date . ' ' . $symbol . ' ' . sprintf('%.8f', $price) . "\n";
array_push($stack, $line);
// Always push a placeholder $/USD for hledger calculations.
// This is so there aren't separate output lines from the
// `--value` calculated and current $/USD holdings.
$line = 'P ' . $date . ' ' . '$' . ' ' . '1' . "\n";
array_push($stack, $line);
$line = 'P ' . $date . ' ' . 'USD' . ' ' . '1' . "\n";
array_push($stack, $line);
//
// HACKS to get USD amount of unsupported upstream coins
//
if ($symbol == 'AAVE') {
$line = 'P ' . $date . ' ' . 'stkAAVE' . ' ' . sprintf('%.8f', $price) . "\n";
array_push($stack, $line);
}
// Hack for array('paxos-standard'=>'USDP')
if ($symbol == 'PAX') {
$line = 'P ' . $date . ' ' . 'USDP' . ' ' . sprintf('%.8f', $price) . "\n";
array_push($stack, $line);
}
// CGLD was changed to CELO at some point
if ($symbol == 'CGLD') {
$line = 'P ' . $date . ' ' . 'CELO' . ' ' . sprintf('%.8f', $price) . "\n";
array_push($stack, $line);
}
// Clobber into most-recent daily average of given year
$average = $price;
}
// Print symbol and most recent price parsed
utils\CLI::print_custom(" \e[32m│\e[0m\n");
utils\CLI::print_custom(" \e[32m├─\e[34m\e[1;3m $symbol\e[0m\n");
utils\CLI::print_custom(" \e[32m│ └─\e[37;2m " . $average . "\e[0m\n");
return $stack;
}
public function getter(string $id, string $timestamp): array
{
$response = [];
$timer = 60; // seconds
$success = false;
while (!$success) {
try {
$response = $this->request($id, $timestamp);
$success = true; // Should throw before this is assigned, alla C++
} catch (\Throwable $e) {
$code = $e->getCode();
if ($code == 429) {
$code = "429 (server rate-limiting)";
}
utils\CLI::print_warning(
"server sent error '" . $code . "' for '$id' ! Trying again in $timer seconds"
);
$i = 1;
$j = 1;
while ($i <= $timer) {
if ($j == 10) {
utils\CLI::print_custom("\e[33;1m+\e[0m");
$j = 0;
} else {
utils\CLI::print_custom("\e[33m.\e[0m");
}
sleep(1);
$i++;
$j++;
}
utils\CLI::print_custom("\n");
}
}
utils\CLI::print_debug($response);
return $response;
}
public function reader(string $symbols): array
{
// Fetched prices for given symbols
$stack = [];
// Timestamp based on given year
$timestamp = $this->make_timestamp($this->get_env()->get_env('API_FETCH_YEAR'));
utils\CLI::print_normal(" ─ Symbols");
foreach ($this->parse_symbols($symbols) as $id => $symbol) {
$parsed = $this->parse_prices(
$symbol,
$this->getter($id, $timestamp)
);
array_push($stack, $this->make_master($symbol, $parsed));
}
utils\CLI::print_custom(" \e[32m│\e[0m\n");
return $stack;
}
public function writer(mixed $data, string $path): void
{
// Cohesive array of all fetched symbols
$stack = [];
// Each element is an array of symbol prices
foreach($data as $symbol) {
foreach($symbol as $price) {
array_push($stack, $price);
}
}
file_put_contents($path, array_unique($stack), LOCK_EX);
}
public function fetcher(): void
{
$master = $this->reader($this->get_env()->get_env('API_PRICES_SYMBOLS'));
$this->writer($master, $this->get_env()->get_env('API_PRICES_PATH'));
}
}
} // namespace docker_finance\prices\internal
//! @since docker-finance 1.0.0
namespace docker_finance\prices
{
require_once('utils/utils.php');

261
container/src/finance/lib/internal/fetch/prices/internal/prices/coingecko.php
View File

@@ -24,19 +24,20 @@
*/
//! @since docker-finance 1.0.0
namespace docker_finance\prices\internal\prices
namespace docker_finance\prices\internal\prices\coingecko
{
require_once('php/vendor/autoload.php'); //!< CoinGecko
require_once('prices/internal/base.php');
require_once('utils/utils.php');
use docker_finance\prices\internal as internal;
use docker_finance\utils as utils;
/**
* @brief CoinGecko cryptocurrency API
* @brief CoinGecko price aggregator API
* @since docker-finance 1.0.0
*/
abstract class CoinGecko extends \docker_finance\prices\internal\API
final class CoinGecko extends internal\Impl
{
private \Codenixsv\CoinGeckoApi\CoinGeckoClient $api; //!< CoinGecko
@@ -48,55 +49,35 @@ namespace docker_finance\prices\internal\prices
}
/**
* @brief Parse given string of 'id/symbol'
* @param string $symbols Expected format: bitcoin/BTC,ethereum/ETH,litecoin/LTC,etc.
*
* @details
*
* CoinGecko caveats:
*
* 1. The reason why ID must be entered and not symbol is because
* coingecko uses same symbol for multiple ID's:
*
* ltc = binance-peg-litecoin
* ltc = litecoin
* eth = ethereum
* eth = ethereum-wormhole
*
* So, pass ID instead of symbol!
*
* 2. Put ticker-symbol *AFTER* ID because hledger's prices are:
*
* - *CASE SENSITIVE*
*
* - *WILL NOT UNDERSTAND THE DIFFERENCE BETWEEN (for example):
* aGUSD and AGUSD* (CoinGecko will return lowercase symbol)
*
* @return array<string> Array of symbols
* @brief REST API request generator
* @param string $id Symbol's ID to request
* @param string $timestamp Timestamp to request
* @return mixed REST API response data
*/
private function parse_symbols(string $symbols): array
protected function request(string $id, string $timestamp): mixed
{
if (!str_contains($this->get_env()->get_env('API_PRICES_SYMBOLS'), '/')) {
utils\CLI::throw_fatal("malformed symbols format");
}
$list = explode(',', $symbols);
$parsed = [];
foreach ($list as $coin) {
list($key, $value) = explode('/', $coin);
$parsed[$key] = $value;
}
return $parsed;
// TODO(afiore): use request_impl() after removing CoinGeckoClient
$response = $this->api->coins()->getMarketChart($id, 'usd', $timestamp);
return $response['prices'];
}
/**
* @brief Parse prices
* @brief Parse given prices for given symbol
* @param string $symbol Given symbol associated with CoinGecko's ID
* @param array<mixed> $prices Array of [N]([timestamp][price])for given year(s)
* @details
*
* Parses historical market data include price, market cap
* and 24h volume (granularity auto)
*
* - Data granularity is automatic (cannot be adjusted)
* - 1 day from current time = 5 minute interval data
* - 1 - 90 days from current time = hourly data
* above 90 days from current time = daily data (00:00 UTC)
*
* @return array<string> Prices for all given symbols
*/
private function parse_prices(string $symbol, array $prices): array
protected function parse_prices(string $symbol, array $prices): array
{
$total = 0;
$prev_date = "";
@@ -183,66 +164,15 @@ namespace docker_finance\prices\internal\prices
$prev_date = $date;
}
//
// Create final stack of journal entries
//
$stack = []; // Final journal entries
$average = 0; // Purely for printing
foreach ($prices_stack as $date => $price) {
// Price journal entry line
$line = 'P ' . $date . ' ' . $symbol . ' ' . sprintf('%.8f', $price) . "\n";
array_push($stack, $line);
// Always push a placeholder $/USD for hledger calculations.
// This is so there aren't separate output lines from the
// `--value` calculated and current $/USD holdings.
$line = 'P ' . $date . ' ' . '$' . ' ' . '1' . "\n";
array_push($stack, $line);
$line = 'P ' . $date . ' ' . 'USD' . ' ' . '1' . "\n";
array_push($stack, $line);
//
// HACKS to get USD amount of unsupported upstream coins
//
if ($symbol == 'AAVE') {
$line = 'P ' . $date . ' ' . 'stkAAVE' . ' ' . sprintf('%.8f', $price) . "\n";
array_push($stack, $line);
}
// Hack for array('paxos-standard'=>'USDP')
if ($symbol == 'PAX') {
$line = 'P ' . $date . ' ' . 'USDP' . ' ' . sprintf('%.8f', $price) . "\n";
array_push($stack, $line);
}
// CGLD was changed to CELO at some point, let's add CELO as well
if ($symbol == 'CGLD') {
$line = 'P ' . $date . ' ' . 'CELO' . ' ' . sprintf('%.8f', $price) . "\n";
array_push($stack, $line);
}
// Clobber into most-recent daily average of given year
$average = $price;
}
// Print symbol and most recent price parsed
utils\CLI::print_custom(" \e[32m│\e[0m\n");
utils\CLI::print_custom(" \e[32m├─\e[34m\e[1;3m $symbol\e[0m\n");
utils\CLI::print_custom(" \e[32m│ └─\e[37;2m " . $average . "\e[0m\n");
return $stack;
return $prices_stack;
}
/**
* @note satisfies date for CoinGecko API
* @brief Make CoinGecko timestamp used in request
* @param string $year Given year
* @return mixed Number of days since given year
*/
private function get_days(string $year): mixed
protected function make_timestamp(string $year): mixed
{
// Number of days back to beginning of given year
if ($year != 'all') {
@@ -269,126 +199,37 @@ namespace docker_finance\prices\internal\prices
// All possible dates
return 'max';
}
}
} // namespace docker_finance\prices\internal\prices\coingecko
/**
* @brief Get prices
* @param string $id CoinGecko's ID of given symbol
* @param string $year Given year(s) to fetch (year or 'max')
* @details
*
* Gets historical market data include price, market cap
* and 24h volume (granularity auto)
*
* - Data granularity is automatic (cannot be adjusted)
* - 1 day from current time = 5 minute interval data
* - 1 - 90 days from current time = hourly data
* above 90 days from current time = daily data (00:00 UTC)
*
* @return array<mixed> Array of [N]([timestamp][price]) for given year(s)
*/
private function get_prices(string $id, string $year): array
//! @since docker-finance 1.0.0
namespace docker_finance\prices\internal\prices
{
require_once('prices/internal/base.php');
require_once('utils/utils.php');
use docker_finance\utils as utils;
/**
* @brief Facade for CoinGecko implementation
* @ingroup php_prices
* @since docker-finance 1.0.0
*/
final class CoinGecko extends \docker_finance\prices\API
{
private coingecko\CoinGecko $api; //!< Internal API
public function __construct(utils\Env $env)
{
$chart = []; // fetched chart
$timer = 60; // seconds
$success = false;
while (!$success) {
try {
$chart = $this->api->coins()->getMarketChart($id, 'usd', $year);
$success = true; // Should throw before this is assigned, alla C++
} catch (\Throwable $e) {
$code = $e->getCode();
if ($code == 429) {
$code = "429 (server rate-limiting)";
}
utils\CLI::print_warning(
"server sent error '" . $code . "' for '$id' ! Trying again in $timer seconds"
);
$i = 1;
$j = 1;
while ($i <= $timer) {
if ($j == 10) {
utils\CLI::print_custom("\e[33;1m+\e[0m");
$j = 0;
} else {
utils\CLI::print_custom("\e[33m.\e[0m");
}
sleep(1);
$i++;
$j++;
}
utils\CLI::print_custom("\n");
}
}
$stack = $chart['prices'];
utils\CLI::print_debug($stack);
return $stack;
}
/**
* @brief Get given symbols
* @return array<int<0, max>, array<string>> Prices for all given symbols
*/
protected function get(string $symbols): array
{
// Fetched prices for given symbols
$stack = [];
utils\CLI::print_normal(" ─ Symbols");
foreach ($this->parse_symbols($symbols) as $id => $symbol) {
array_push($stack, $this->parse_prices(
$symbol,
$this->get_prices(
$id,
$this->get_days($this->get_env()->get_env('API_FETCH_YEAR'))
)
));
}
utils\CLI::print_custom(" \e[32m│\e[0m\n");
return $stack;
}
/**
* @brief Write prices master file
* @param array<array<string>> $data Data to write
* @param string $path Path to file
* @note External impl *MUST* append and sort to every applicable year
* @warning Clobbers external master price file
*/
protected function write(array $data, string $path): void
{
// Cohesive array of all fetched symbols
$stack = [];
// Each element is an array of symbol prices
foreach($data as $symbol) {
foreach($symbol as $price) {
array_push($stack, $price);
}
}
file_put_contents($path, array_unique($stack), LOCK_EX);
$this->api = new coingecko\CoinGecko($env);
}
public function fetch(): void
{
$prices = $this->get($this->get_env()->get_env('API_PRICES_SYMBOLS'));
$this->write($prices, $this->get_env()->get_env('API_PRICES_PATH'));
$this->api->fetcher();
}
}
/**
* @brief Cryptocurrency prices API
* @since docker-finance 1.0.0
*/
final class Crypto extends CoinGecko
{
}
} // namespace docker_finance\prices\internal\prices
# vim: sw=4 sts=4 si ai et