mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2025-02-25 18:45:27 -06:00
577 lines
24 KiB
PHP
577 lines
24 KiB
PHP
<?php
|
|
/*
|
|
* ForceDecimalSize.php
|
|
* Copyright (c) 2023 james@firefly-iii.org
|
|
*
|
|
* This file is part of Firefly III (https://github.com/firefly-iii).
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as
|
|
* published by the Free Software Foundation, either version 3 of the
|
|
* License, or (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Affero General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace FireflyIII\Console\Commands\System;
|
|
|
|
use FireflyIII\Console\Commands\ShowsFriendlyMessages;
|
|
use FireflyIII\Exceptions\FireflyException;
|
|
use FireflyIII\Models\Account;
|
|
use FireflyIII\Models\AutoBudget;
|
|
use FireflyIII\Models\AvailableBudget;
|
|
use FireflyIII\Models\Bill;
|
|
use FireflyIII\Models\BudgetLimit;
|
|
use FireflyIII\Models\PiggyBank;
|
|
use FireflyIII\Models\PiggyBankEvent;
|
|
use FireflyIII\Models\PiggyBankRepetition;
|
|
use FireflyIII\Models\RecurrenceTransaction;
|
|
use FireflyIII\Models\Transaction;
|
|
use FireflyIII\Models\TransactionCurrency;
|
|
use Illuminate\Console\Command;
|
|
use Illuminate\Database\Eloquent\Builder;
|
|
use Illuminate\Database\Eloquent\Model;
|
|
use Illuminate\Support\Collection;
|
|
use Illuminate\Support\Facades\DB;
|
|
|
|
/**
|
|
* Class ForceDecimalSize
|
|
*
|
|
* This command was inspired by https://github.com/elliot-gh. It will check all amount fields
|
|
* and their values and correct them to the correct number of decimal places. This fixes issues where
|
|
* Firefly III would store 0.01 as 0.01000000000000000020816681711721685132943093776702880859375.
|
|
*/
|
|
class ForceDecimalSize extends Command
|
|
{
|
|
use ShowsFriendlyMessages;
|
|
|
|
protected $description = 'This command resizes DECIMAL columns in MySQL or PostgreSQL and correct amounts (only MySQL).';
|
|
protected $signature = 'firefly-iii:force-decimal-size';
|
|
private string $cast;
|
|
private array $classes
|
|
= [
|
|
'accounts' => Account::class,
|
|
'auto_budgets' => AutoBudget::class,
|
|
'available_budgets' => AvailableBudget::class,
|
|
'bills' => Bill::class,
|
|
'budget_limits' => BudgetLimit::class,
|
|
'piggy_bank_events' => PiggyBankEvent::class,
|
|
'piggy_bank_repetitions' => PiggyBankRepetition::class,
|
|
'piggy_banks' => PiggyBank::class,
|
|
'recurrences_transactions' => RecurrenceTransaction::class,
|
|
'transactions' => Transaction::class,
|
|
];
|
|
|
|
private string $operator;
|
|
private string $regularExpression;
|
|
private array $tables
|
|
= [
|
|
'accounts' => ['virtual_balance'],
|
|
'auto_budgets' => ['amount'],
|
|
'available_budgets' => ['amount'],
|
|
'bills' => ['amount_min', 'amount_max'],
|
|
'budget_limits' => ['amount'],
|
|
'currency_exchange_rates' => ['rate', 'user_rate'],
|
|
'limit_repetitions' => ['amount'],
|
|
'piggy_bank_events' => ['amount'],
|
|
'piggy_bank_repetitions' => ['currentamount'],
|
|
'piggy_banks' => ['targetamount'],
|
|
'recurrences_transactions' => ['amount', 'foreign_amount'],
|
|
'transactions' => ['amount', 'foreign_amount'],
|
|
];
|
|
|
|
/**
|
|
* Execute the console command.
|
|
*
|
|
*/
|
|
public function handle(): int
|
|
{
|
|
app('log')->debug('Now in ForceDecimalSize::handle()');
|
|
$this->determineDatabaseType();
|
|
|
|
$this->friendlyError('Running this command is dangerous and can cause data loss.');
|
|
$this->friendlyError('Please do not continue.');
|
|
$question = $this->confirm('Do you want to continue?');
|
|
if (true === $question) {
|
|
$this->correctAmounts();
|
|
$this->updateDecimals();
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* @return void
|
|
*/
|
|
private function determineDatabaseType(): void
|
|
{
|
|
// switch stuff based on database connection:
|
|
$this->operator = 'REGEXP';
|
|
$this->regularExpression = '\'\\\\.[\\\\d]{%d}[1-9]+\'';
|
|
$this->cast = 'CHAR';
|
|
if ('pgsql' === config('database.default')) {
|
|
$this->operator = 'SIMILAR TO';
|
|
$this->regularExpression = '\'%%\.[\d]{%d}[1-9]+%%\'';
|
|
$this->cast = 'TEXT';
|
|
}
|
|
if ('sqlite' === config('database.default')) {
|
|
$this->regularExpression = '"\\.[\d]{%d}[1-9]+"';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method checks if a basic check can be done or if it needs to be complicated.
|
|
*
|
|
* @return void
|
|
*/
|
|
private function correctAmounts(): void
|
|
{
|
|
// if sqlite, add function?
|
|
if ('sqlite' === (string)config('database.default')) {
|
|
DB::connection()->getPdo()->sqliteCreateFunction('REGEXP', function ($pattern, $value) {
|
|
mb_regex_encoding('UTF-8');
|
|
$pattern = trim($pattern, '"');
|
|
|
|
return (false !== mb_ereg($pattern, (string)$value)) ? 1 : 0;
|
|
});
|
|
}
|
|
|
|
if (!in_array((string)config('database.default'), ['mysql', 'pgsql', 'sqlite'], true)) {
|
|
$this->friendlyWarning(sprintf('Skip correcting amounts, does not support "%s"...', (string)config('database.default')));
|
|
|
|
return;
|
|
}
|
|
$this->correctAmountsByCurrency();
|
|
}
|
|
|
|
/**
|
|
* This method loops all enabled currencies and then calls the method that will fix all objects in this currency.
|
|
*
|
|
* @return void
|
|
* @throws FireflyException
|
|
*/
|
|
private function correctAmountsByCurrency(): void
|
|
{
|
|
/** @var Collection $enabled */
|
|
$enabled = TransactionCurrency::whereEnabled(1)->get();
|
|
/** @var TransactionCurrency $currency */
|
|
foreach ($enabled as $currency) {
|
|
$this->correctByCurrency($currency);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method loops the available tables that may need fixing, and calls for the right method that can fix them.
|
|
*
|
|
* @param TransactionCurrency $currency
|
|
*
|
|
* @return void
|
|
* @throws FireflyException
|
|
*/
|
|
private function correctByCurrency(TransactionCurrency $currency): void
|
|
{
|
|
/**
|
|
* @var string $name
|
|
* @var array $fields
|
|
*/
|
|
foreach ($this->tables as $name => $fields) {
|
|
switch ($name) { // @phpstan-ignore-line
|
|
default:
|
|
$message = sprintf('Cannot handle table "%s"', $name);
|
|
$this->friendlyError($message);
|
|
throw new FireflyException($message);
|
|
case 'accounts':
|
|
$this->correctAccountAmounts($currency, $fields);
|
|
break;
|
|
case 'auto_budgets':
|
|
case 'available_budgets':
|
|
case 'bills':
|
|
case 'budget_limits':
|
|
case 'recurrences_transactions':
|
|
$this->correctGeneric($currency, $name);
|
|
break;
|
|
case 'currency_exchange_rates':
|
|
case 'limit_repetitions':
|
|
// do nothing
|
|
break;
|
|
case 'piggy_bank_events':
|
|
$this->correctPiggyEventAmounts($currency, $fields);
|
|
break;
|
|
case 'piggy_bank_repetitions':
|
|
$this->correctPiggyRepetitionAmounts($currency, $fields);
|
|
break;
|
|
case 'piggy_banks':
|
|
$this->correctPiggyAmounts($currency, $fields);
|
|
break;
|
|
case 'transactions':
|
|
$this->correctTransactionAmounts($currency);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method loops over all accounts and validates the amounts.
|
|
*
|
|
* @param TransactionCurrency $currency
|
|
* @param array $fields
|
|
*
|
|
* @return void
|
|
*/
|
|
private function correctAccountAmounts(TransactionCurrency $currency, array $fields): void
|
|
{
|
|
$operator = $this->operator;
|
|
$cast = $this->cast;
|
|
$regularExpression = $this->regularExpression;
|
|
|
|
/** @var Builder $query */
|
|
$query = Account::leftJoin('account_meta', 'accounts.id', '=', 'account_meta.account_id')
|
|
->where('account_meta.name', 'currency_id')
|
|
->where('account_meta.data', json_encode((string)$currency->id));
|
|
$query->where(static function (Builder $q) use ($fields, $currency, $operator, $cast, $regularExpression) {
|
|
foreach ($fields as $field) {
|
|
$q->orWhere(
|
|
DB::raw(sprintf('CAST(accounts.%s AS %s)', $field, $cast)),
|
|
$operator,
|
|
DB::raw(sprintf($regularExpression, $currency->decimal_places))
|
|
);
|
|
}
|
|
});
|
|
$result = $query->get(['accounts.*']);
|
|
if (0 === $result->count()) {
|
|
$this->friendlyPositive(sprintf('All accounts in %s are OK', $currency->code));
|
|
|
|
return;
|
|
}
|
|
/** @var Account $account */
|
|
foreach ($result as $account) {
|
|
foreach ($fields as $field) {
|
|
$value = $account->$field;
|
|
if (null === $value) {
|
|
continue;
|
|
}
|
|
// fix $field by rounding it down correctly.
|
|
$pow = pow(10, (int)$currency->decimal_places);
|
|
$correct = bcdiv((string)round($value * $pow), (string)$pow, 12);
|
|
$this->friendlyInfo(sprintf('Account #%d has %s with value "%s", this has been corrected to "%s".', $account->id, $field, $value, $correct));
|
|
Account::find($account->id)->update([$field => $correct]);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method fixes all auto budgets in currency $currency.
|
|
*
|
|
* @param TransactionCurrency $currency
|
|
* @param string $table
|
|
*
|
|
* @return void
|
|
*/
|
|
private function correctGeneric(TransactionCurrency $currency, string $table): void
|
|
{
|
|
$class = $this->classes[$table];
|
|
$fields = $this->tables[$table];
|
|
$operator = $this->operator;
|
|
$cast = $this->cast;
|
|
$regularExpression = $this->regularExpression;
|
|
|
|
/** @var Builder $query */
|
|
$query = $class::where('transaction_currency_id', $currency->id)->where(
|
|
static function (Builder $q) use ($fields, $currency, $operator, $cast, $regularExpression) {
|
|
foreach ($fields as $field) {
|
|
$q->orWhere(
|
|
DB::raw(sprintf('CAST(%s AS %s)', $field, $cast)),
|
|
$operator,
|
|
DB::raw(sprintf($regularExpression, $currency->decimal_places))
|
|
);
|
|
}
|
|
}
|
|
);
|
|
|
|
$result = $query->get(['*']);
|
|
if (0 === $result->count()) {
|
|
$this->friendlyPositive(sprintf('All %s in %s are OK', $table, $currency->code));
|
|
|
|
return;
|
|
}
|
|
/** @var Model $item */
|
|
foreach ($result as $item) {
|
|
foreach ($fields as $field) {
|
|
$value = $item->$field;
|
|
if (null === $value) {
|
|
continue;
|
|
}
|
|
// fix $field by rounding it down correctly.
|
|
$pow = pow(10, (int)$currency->decimal_places);
|
|
$correct = bcdiv((string)round($value * $pow), (string)$pow, 12);
|
|
$this->friendlyWarning(sprintf('%s #%d has %s with value "%s", this has been corrected to "%s".', $table, $item->id, $field, $value, $correct));
|
|
$class::find($item->id)->update([$field => $correct]);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method fixes all piggy bank events in currency $currency.
|
|
*
|
|
* @param TransactionCurrency $currency
|
|
* @param array $fields
|
|
*
|
|
* @return void
|
|
*/
|
|
private function correctPiggyEventAmounts(TransactionCurrency $currency, array $fields): void
|
|
{
|
|
$operator = $this->operator;
|
|
$cast = $this->cast;
|
|
$regularExpression = $this->regularExpression;
|
|
|
|
/** @var Builder $query */
|
|
$query = PiggyBankEvent::leftJoin('piggy_banks', 'piggy_bank_events.piggy_bank_id', '=', 'piggy_banks.id')
|
|
->leftJoin('accounts', 'piggy_banks.account_id', '=', 'accounts.id')
|
|
->leftJoin('account_meta', 'accounts.id', '=', 'account_meta.account_id')
|
|
->where('account_meta.name', 'currency_id')
|
|
->where('account_meta.data', json_encode((string)$currency->id))
|
|
->where(static function (Builder $q) use ($fields, $currency, $cast, $operator, $regularExpression) {
|
|
foreach ($fields as $field) {
|
|
$q->orWhere(
|
|
DB::raw(sprintf('CAST(piggy_bank_events.%s AS %s)', $field, $cast)),
|
|
$operator,
|
|
DB::raw(sprintf($regularExpression, $currency->decimal_places))
|
|
);
|
|
}
|
|
});
|
|
|
|
$result = $query->get(['piggy_bank_events.*']);
|
|
if (0 === $result->count()) {
|
|
$this->friendlyPositive(sprintf('All piggy bank events in %s are OK', $currency->code));
|
|
|
|
return;
|
|
}
|
|
/** @var PiggyBankEvent $item */
|
|
foreach ($result as $item) {
|
|
foreach ($fields as $field) {
|
|
$value = $item->$field;
|
|
if (null === $value) {
|
|
continue;
|
|
}
|
|
// fix $field by rounding it down correctly.
|
|
$pow = pow(10, (int)$currency->decimal_places);
|
|
$correct = bcdiv((string)round($value * $pow), (string)$pow, 12);
|
|
$this->friendlyWarning(
|
|
sprintf('Piggy bank event #%d has %s with value "%s", this has been corrected to "%s".', $item->id, $field, $value, $correct)
|
|
);
|
|
PiggyBankEvent::find($item->id)->update([$field => $correct]);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method fixes all piggy bank repetitions in currency $currency.
|
|
*
|
|
* @param TransactionCurrency $currency
|
|
* @param array $fields
|
|
*
|
|
* @return void
|
|
*/
|
|
private function correctPiggyRepetitionAmounts(TransactionCurrency $currency, array $fields)
|
|
{
|
|
$operator = $this->operator;
|
|
$cast = $this->cast;
|
|
$regularExpression = $this->regularExpression;
|
|
// select all piggy bank repetitions with this currency and issue.
|
|
/** @var Builder $query */
|
|
$query = PiggyBankRepetition::leftJoin('piggy_banks', 'piggy_bank_repetitions.piggy_bank_id', '=', 'piggy_banks.id')
|
|
->leftJoin('accounts', 'piggy_banks.account_id', '=', 'accounts.id')
|
|
->leftJoin('account_meta', 'accounts.id', '=', 'account_meta.account_id')
|
|
->where('account_meta.name', 'currency_id')
|
|
->where('account_meta.data', json_encode((string)$currency->id))
|
|
->where(static function (Builder $q) use ($fields, $currency, $operator, $cast, $regularExpression) {
|
|
foreach ($fields as $field) {
|
|
$q->orWhere(
|
|
DB::raw(sprintf('CAST(piggy_bank_repetitions.%s AS %s)', $field, $cast)),
|
|
$operator,
|
|
DB::raw(sprintf($regularExpression, $currency->decimal_places))
|
|
);
|
|
}
|
|
});
|
|
|
|
$result = $query->get(['piggy_bank_repetitions.*']);
|
|
if (0 === $result->count()) {
|
|
$this->friendlyPositive(sprintf('All piggy bank repetitions in %s', $currency->code));
|
|
|
|
return;
|
|
}
|
|
/** @var PiggyBankRepetition $item */
|
|
foreach ($result as $item) {
|
|
foreach ($fields as $field) {
|
|
$value = $item->$field;
|
|
if (null === $value) {
|
|
continue;
|
|
}
|
|
// fix $field by rounding it down correctly.
|
|
$pow = pow(10, (int)$currency->decimal_places);
|
|
$correct = bcdiv((string)round($value * $pow), (string)$pow, 12);
|
|
$this->friendlyWarning(
|
|
sprintf('Piggy bank repetition #%d has %s with value "%s", this has been corrected to "%s".', $item->id, $field, $value, $correct)
|
|
);
|
|
PiggyBankRepetition::find($item->id)->update([$field => $correct]);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method fixes all piggy banks in currency $currency.
|
|
*
|
|
* @param TransactionCurrency $currency
|
|
* @param array $fields
|
|
*
|
|
* @return void
|
|
*/
|
|
private function correctPiggyAmounts(TransactionCurrency $currency, array $fields): void
|
|
{
|
|
$operator = $this->operator;
|
|
$cast = $this->cast;
|
|
$regularExpression = $this->regularExpression;
|
|
|
|
/** @var Builder $query */
|
|
$query = PiggyBank::leftJoin('accounts', 'piggy_banks.account_id', '=', 'accounts.id')
|
|
->leftJoin('account_meta', 'accounts.id', '=', 'account_meta.account_id')
|
|
->where('account_meta.name', 'currency_id')
|
|
->where('account_meta.data', json_encode((string)$currency->id))
|
|
->where(static function (Builder $q) use ($fields, $currency, $operator, $cast, $regularExpression) {
|
|
foreach ($fields as $field) {
|
|
$q->orWhere(
|
|
DB::raw(sprintf('CAST(piggy_banks.%s AS %s)', $field, $cast)),
|
|
$operator,
|
|
DB::raw(sprintf($regularExpression, $currency->decimal_places))
|
|
);
|
|
}
|
|
});
|
|
|
|
$result = $query->get(['piggy_banks.*']);
|
|
if (0 === $result->count()) {
|
|
$this->friendlyPositive(sprintf('All piggy banks in %s are OK', $currency->code));
|
|
|
|
return;
|
|
}
|
|
/** @var PiggyBank $item */
|
|
foreach ($result as $item) {
|
|
foreach ($fields as $field) {
|
|
$value = $item->$field;
|
|
if (null === $value) {
|
|
continue;
|
|
}
|
|
// fix $field by rounding it down correctly.
|
|
$pow = pow(10, (int)$currency->decimal_places);
|
|
$correct = bcdiv((string)round($value * $pow), (string)$pow, 12);
|
|
$this->friendlyWarning(sprintf('Piggy bank #%d has %s with value "%s", this has been corrected to "%s".', $item->id, $field, $value, $correct));
|
|
PiggyBank::find($item->id)->update([$field => $correct]);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method fixes all transactions in currency $currency.
|
|
*
|
|
* @param TransactionCurrency $currency
|
|
*
|
|
* @return void
|
|
*/
|
|
private function correctTransactionAmounts(TransactionCurrency $currency): void
|
|
{
|
|
// select all transactions with this currency and issue.
|
|
/** @var Builder $query */
|
|
$query = Transaction::where('transaction_currency_id', $currency->id)->where(
|
|
DB::raw(sprintf('CAST(amount as %s)', $this->cast)),
|
|
$this->operator,
|
|
DB::raw(sprintf($this->regularExpression, $currency->decimal_places))
|
|
);
|
|
|
|
$result = $query->get(['transactions.*']);
|
|
if (0 === $result->count()) {
|
|
$this->friendlyPositive(sprintf('All transactions in %s are OK', $currency->code));
|
|
}
|
|
|
|
/** @var Transaction $item */
|
|
foreach ($result as $item) {
|
|
$value = $item->amount;
|
|
if (null === $value) {
|
|
continue;
|
|
}
|
|
// fix $field by rounding it down correctly.
|
|
$pow = (float)pow(10, (int)$currency->decimal_places);
|
|
$correct = bcdiv((string)round((float)$value * $pow), (string)$pow, 12);
|
|
$this->friendlyWarning(sprintf('Transaction #%d has amount with value "%s", this has been corrected to "%s".', $item->id, $value, $correct));
|
|
Transaction::find($item->id)->update(['amount' => $correct]);
|
|
}
|
|
|
|
// select all transactions with this FOREIGN currency and issue.
|
|
/** @var Builder $query */
|
|
$query = Transaction::where('foreign_currency_id', $currency->id)->where(
|
|
DB::raw(sprintf('CAST(foreign_amount as %s)', $this->cast)),
|
|
$this->operator,
|
|
DB::raw(sprintf($this->regularExpression, $currency->decimal_places))
|
|
);
|
|
|
|
$result = $query->get(['*']);
|
|
if (0 === $result->count()) {
|
|
$this->friendlyPositive(sprintf('All transactions in foreign currency %s are OK', $currency->code));
|
|
|
|
return;
|
|
}
|
|
/** @var Transaction $item */
|
|
foreach ($result as $item) {
|
|
$value = $item->foreign_amount;
|
|
if (null === $value) {
|
|
continue;
|
|
}
|
|
// fix $field by rounding it down correctly.
|
|
$pow = (float)pow(10, (int)$currency->decimal_places);
|
|
$correct = bcdiv((string)round((float)$value * $pow), (string)$pow, 12);
|
|
$this->friendlyWarning(
|
|
sprintf('Transaction #%d has foreign amount with value "%s", this has been corrected to "%s".', $item->id, $value, $correct)
|
|
);
|
|
Transaction::find($item->id)->update(['foreign_amount' => $correct]);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @return void
|
|
*/
|
|
private function updateDecimals(): void
|
|
{
|
|
$this->friendlyInfo('Going to force the size of DECIMAL columns. Please hold.');
|
|
$type = (string)config('database.default');
|
|
|
|
/**
|
|
* @var string $name
|
|
* @var array $fields
|
|
*/
|
|
foreach ($this->tables as $name => $fields) {
|
|
/** @var string $field */
|
|
foreach ($fields as $field) {
|
|
$this->friendlyLine(sprintf('Updating table "%s", field "%s"...', $name, $field));
|
|
if ('pgsql' === $type) {
|
|
DB::select(sprintf('ALTER TABLE %s ALTER COLUMN %s TYPE DECIMAL(32,12);', $name, $field));
|
|
sleep(1);
|
|
return;
|
|
}
|
|
if ('mysql' === $type) {
|
|
DB::select(sprintf('ALTER TABLE %s CHANGE COLUMN %s %s DECIMAL(32, 12);', $name, $field, $field));
|
|
sleep(1);
|
|
return;
|
|
}
|
|
$this->friendlyError(sprintf('Cannot handle database type "%s".', $type));
|
|
|
|
}
|
|
}
|
|
}
|
|
}
|