From cba1213dd150783dbc3e4a268dd2946b9985df60 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sat, 6 Jul 2024 15:42:50 +0200 Subject: [PATCH] Better cache --- .../Http/Api/ExchangeRateConverter.php | 126 ++++++++++-------- 1 file changed, 69 insertions(+), 57 deletions(-) diff --git a/app/Support/Http/Api/ExchangeRateConverter.php b/app/Support/Http/Api/ExchangeRateConverter.php index 9b1004845d..0299e112fd 100644 --- a/app/Support/Http/Api/ExchangeRateConverter.php +++ b/app/Support/Http/Api/ExchangeRateConverter.php @@ -29,6 +29,7 @@ use FireflyIII\Exceptions\FireflyException; use FireflyIII\Models\CurrencyExchangeRate; use FireflyIII\Models\TransactionCurrency; use FireflyIII\Support\CacheProperties; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Log; /** @@ -48,10 +49,10 @@ class ExchangeRateConverter */ public function convert(TransactionCurrency $from, TransactionCurrency $to, Carbon $date, string $amount): string { - if(false === config('cer.enabled')) { + if (false === config('cer.enabled')) { + Log::debug('ExchangeRateConverter: disabled, return amount as is.'); return $amount; } - Log::debug('convert()'); $rate = $this->getCurrencyRate($from, $to, $date); return bcmul($amount, $rate); @@ -62,10 +63,10 @@ class ExchangeRateConverter */ public function getCurrencyRate(TransactionCurrency $from, TransactionCurrency $to, Carbon $date): string { - if(false === config('cer.enabled')) { + if (false === config('cer.enabled')) { + Log::debug('ExchangeRateConverter: disabled, return "1".'); return '1'; } - Log::debug('getCurrencyRate()'); $rate = $this->getRate($from, $to, $date); return '0' === $rate ? '1' : $rate; @@ -76,25 +77,33 @@ class ExchangeRateConverter */ private function getRate(TransactionCurrency $from, TransactionCurrency $to, Carbon $date): string { - Log::debug('getRate()'); - if ($this->isPrepared && $this->noPreparedRates) { - $fallback = $this->fallback[$from->id][$to->id] ?? '0'; - Log::debug(sprintf('Return fallback rate from #%d to #%d on %s: %s', $from->id, $to->id, $date->format('Y-m-d'), $fallback)); + $key = $this->getCacheKey($from, $to, $date); + $res = Cache::get($key, null); - return $fallback; + // find in cache + if (null !== $res) { + Log::debug(sprintf('ExchangeRateConverter: Return cached rate from #%d to #%d on %s.', $from->id, $to->id, $date->format('Y-m-d'))); + return $res; } - // first attempt: - $rate = $this->getFromDB($from->id, $to->id, $date->format('Y-m-d')); + + // find in database + $rate = $this->getFromDB($from->id, $to->id, $date->format('Y-m-d')); if (null !== $rate) { + Cache::forever($key, $rate); + Log::debug(sprintf('ExchangeRateConverter: Return DB rate from #%d to #%d on %s.', $from->id, $to->id, $date->format('Y-m-d'))); return $rate; } - // no result. perhaps the other way around? - $rate = $this->getFromDB($to->id, $from->id, $date->format('Y-m-d')); + + // find reverse in database + $rate = $this->getFromDB($to->id, $from->id, $date->format('Y-m-d')); if (null !== $rate) { - return bcdiv('1', $rate); + $rate = bcdiv('1', $rate); + Cache::forever($key, $rate); + Log::debug(sprintf('ExchangeRateConverter: Return DB rate from #%d to #%d on %s.', $from->id, $to->id, $date->format('Y-m-d'))); + return $rate; } - // if nothing in place, fall back on the rate for $from to EUR + // fallback scenario. $first = $this->getEuroRate($from, $date); $second = $this->getEuroRate($to, $date); @@ -102,65 +111,65 @@ class ExchangeRateConverter if (0 === bccomp('0', $first) || 0 === bccomp('0', $second)) { Log::warning(sprintf('$first is "%s" and $second is "%s"', $first, $second)); - return '0'; + return '1'; } $second = bcdiv('1', $second); - - return bcmul($first, $second); + $rate = bcmul($first, $second); + Log::debug(sprintf('ExchangeRateConverter: Return DB rate from #%d to #%d on %s.', $from->id, $to->id, $date->format('Y-m-d'))); + Cache::forever($key, $rate); + return $rate; } private function getFromDB(int $from, int $to, string $date): ?string { - Log::debug('getFromDB()'); if ($from === $to) { return '1'; } - $key = sprintf('cer-%d-%d-%s', $from, $to, $date); + $key = sprintf('cer-%d-%d-%s', $from, $to, $date); // perhaps the rate has been cached during this particular run $preparedRate = $this->prepared[$date][$from][$to] ?? null; if (null !== $preparedRate && 0 !== bccomp('0', $preparedRate)) { - Log::debug(sprintf('Found prepared rate from #%d to #%d on %s.', $from, $to, $date)); + Log::debug(sprintf('ExchangeRateConverter: Found prepared rate from #%d to #%d on %s.', $from, $to, $date)); return $preparedRate; } - $cache = new CacheProperties(); + $cache = new CacheProperties(); $cache->addProperty($key); if ($cache->has()) { $rate = $cache->get(); if ('' === $rate) { return null; } - Log::debug(sprintf('Found cached rate from #%d to #%d on %s.', $from, $to, $date)); + Log::debug(sprintf('ExchangeRateConverter: Found !cached! rate from #%d to #%d on %s.', $from, $to, $date)); return $rate; } /** @var null|CurrencyExchangeRate $result */ - $result = auth()->user() - ->currencyExchangeRates() - ->where('from_currency_id', $from) - ->where('to_currency_id', $to) - ->where('date', '<=', $date) - ->orderBy('date', 'DESC') - ->first() - ; + $result = auth()->user() + ->currencyExchangeRates() + ->where('from_currency_id', $from) + ->where('to_currency_id', $to) + ->where('date', '<=', $date) + ->orderBy('date', 'DESC') + ->first(); ++$this->queryCount; - $rate = (string)$result?->rate; + $rate = (string) $result?->rate; if ('' === $rate) { - app('log')->debug(sprintf('Found no rate for #%d->#%d (%s) in the DB.', $from, $to, $date)); + app('log')->debug(sprintf('ExchangeRateConverter: Found no rate for #%d->#%d (%s) in the DB.', $from, $to, $date)); return null; } if (0 === bccomp('0', $rate)) { - app('log')->debug(sprintf('Found rate for #%d->#%d (%s) in the DB, but it\'s zero.', $from, $to, $date)); + app('log')->debug(sprintf('ExchangeRateConverter: Found rate for #%d->#%d (%s) in the DB, but it\'s zero.', $from, $to, $date)); return null; } - app('log')->debug(sprintf('Found rate for #%d->#%d (%s) in the DB: %s.', $from, $to, $date, $rate)); + app('log')->debug(sprintf('ExchangeRateConverter: Found rate for #%d->#%d (%s) in the DB: %s.', $from, $to, $date, $rate)); $cache->store($rate); // if the rate has not been cached during this particular run, save it @@ -184,18 +193,17 @@ class ExchangeRateConverter */ private function getEuroRate(TransactionCurrency $currency, Carbon $date): string { - Log::debug('getEuroRate()'); $euroId = $this->getEuroId(); if ($euroId === $currency->id) { return '1'; } - $rate = $this->getFromDB($currency->id, $euroId, $date->format('Y-m-d')); + $rate = $this->getFromDB($currency->id, $euroId, $date->format('Y-m-d')); if (null !== $rate) { // app('log')->debug(sprintf('Rate for %s to EUR is %s.', $currency->code, $rate)); return $rate; } - $rate = $this->getFromDB($euroId, $currency->id, $date->format('Y-m-d')); + $rate = $this->getFromDB($euroId, $currency->id, $date->format('Y-m-d')); if (null !== $rate) { return bcdiv('1', $rate); // app('log')->debug(sprintf('Inverted rate for %s to EUR is %s.', $currency->code, $rate)); @@ -204,7 +212,7 @@ class ExchangeRateConverter // grab backup values from config file: $backup = config(sprintf('cer.rates.%s', $currency->code)); if (null !== $backup) { - return bcdiv('1', (string)$backup); + return bcdiv('1', (string) $backup); // app('log')->debug(sprintf('Backup rate for %s to EUR is %s.', $currency->code, $backup)); // return $backup; } @@ -222,9 +230,9 @@ class ExchangeRateConverter $cache = new CacheProperties(); $cache->addProperty('cer-euro-id'); if ($cache->has()) { - return (int)$cache->get(); + return (int) $cache->get(); } - $euro = TransactionCurrency::whereCode('EUR')->first(); + $euro = TransactionCurrency::whereCode('EUR')->first(); ++$this->queryCount; if (null === $euro) { throw new FireflyException('Cannot find EUR in system, cannot do currency conversion.'); @@ -239,21 +247,20 @@ class ExchangeRateConverter */ public function prepare(TransactionCurrency $from, TransactionCurrency $to, Carbon $start, Carbon $end): void { - if(false === config('cer.enabled')) { + if (false === config('cer.enabled')) { return; } Log::debug('prepare()'); $start->startOfDay(); $end->endOfDay(); Log::debug(sprintf('Preparing for %s to %s between %s and %s', $from->code, $to->code, $start->format('Y-m-d'), $end->format('Y-m-d'))); - $set = auth()->user() - ->currencyExchangeRates() - ->where('from_currency_id', $from->id) - ->where('to_currency_id', $to->id) - ->where('date', '<=', $end->format('Y-m-d')) - ->where('date', '>=', $start->format('Y-m-d')) - ->orderBy('date', 'DESC')->get() - ; + $set = auth()->user() + ->currencyExchangeRates() + ->where('from_currency_id', $from->id) + ->where('to_currency_id', $to->id) + ->where('date', '<=', $end->format('Y-m-d')) + ->where('date', '>=', $start->format('Y-m-d')) + ->orderBy('date', 'DESC')->get(); ++$this->queryCount; if (0 === $set->count()) { Log::debug('No prepared rates found in this period, use the fallback'); @@ -267,10 +274,10 @@ class ExchangeRateConverter $this->isPrepared = true; // so there is a fallback just in case. Now loop the set of rates we DO have. - $temp = []; - $count = 0; + $temp = []; + $count = 0; foreach ($set as $rate) { - $date = $rate->date->format('Y-m-d'); + $date = $rate->date->format('Y-m-d'); $temp[$date] ??= [ $from->id => [ $to->id => $rate->rate, @@ -279,11 +286,11 @@ class ExchangeRateConverter ++$count; } Log::debug(sprintf('Found %d rates in this period.', $count)); - $currentStart = clone $start; + $currentStart = clone $start; while ($currentStart->lte($end)) { - $currentDate = $currentStart->format('Y-m-d'); + $currentDate = $currentStart->format('Y-m-d'); $this->prepared[$currentDate] ??= []; - $fallback = $temp[$currentDate][$from->id][$to->id] ?? $this->fallback[$from->id][$to->id] ?? '0'; + $fallback = $temp[$currentDate][$from->id][$to->id] ?? $this->fallback[$from->id][$to->id] ?? '0'; if (0 === count($this->prepared[$currentDate]) && 0 !== bccomp('0', $fallback)) { // fill from temp or fallback or from temp (see before) $this->prepared[$currentDate][$from->id][$to->id] = $fallback; @@ -314,9 +321,14 @@ class ExchangeRateConverter public function summarize(): void { - if(false === config('cer.enabled')) { + if (false === config('cer.enabled')) { return; } Log::debug(sprintf('ExchangeRateConverter ran %d queries.', $this->queryCount)); } + + private function getCacheKey(TransactionCurrency $from, TransactionCurrency $to, Carbon $date): string + { + return sprintf('cer-%d-%d-%s', $from->id, $to->id, $date->format('Y-m-d')); + } }