FF3 will apply rules when importing from bunq #1443

This commit is contained in:
James Cole 2018-06-01 05:49:33 +02:00
parent 3fbe851a0b
commit df87d03f32
4 changed files with 82 additions and 698 deletions

View File

@ -1,389 +0,0 @@
<?php
/**
* ImportStorage.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Storage;
use Exception;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Factory\TransactionJournalFactory;
use FireflyIII\Import\Object\ImportJournal;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Journal\JournalRepositoryInterface;
use Illuminate\Support\Collection;
use Log;
use Preferences;
/**
* @codeCoverageIgnore
* @deprecated
* Is capable of storing individual ImportJournal objects.
* Adds 7 steps per object stored:
* 1. get all import data from import journal
* 2. is not a duplicate
* 3. create the journal
* 4. store journal
* 5. run rules
* 6. run bills
* 7. finished storing object
*
* Class ImportStorage.
*/
class ImportStorage
{
use ImportSupport;
/** @var Collection */
public $errors;
/** @var Collection */
public $journals;
/** @var int */
protected $defaultCurrencyId = 1;
/** @var ImportJob */
protected $job; // yes, hard coded
/** @var JournalRepositoryInterface */
protected $journalRepository;
/** @var ImportJobRepositoryInterface */
protected $repository;
/** @var Collection */
protected $rules;
/** @var bool */
private $applyRules = false;
/** @var string */
private $dateFormat = 'Ymd';
/** @var TransactionJournalFactory */
private $factory;
/** @var Collection */
private $objects;
/** @var int */
private $total = 0;
/** @var array */
private $transfers = [];
/**
* ImportStorage constructor.
*/
public function __construct()
{
$this->objects = new Collection;
$this->journals = new Collection;
$this->errors = new Collection;
}
/**
* @param string $dateFormat
*/
public function setDateFormat(string $dateFormat)
{
$this->dateFormat = $dateFormat;
}
/**
* @param ImportJob $job
*/
public function setJob(ImportJob $job)
{
$this->repository = app(ImportJobRepositoryInterface::class);
$this->journalRepository = app(JournalRepositoryInterface::class);
$this->repository->setUser($job->user);
$this->journalRepository->setUser($job->user);
$this->factory = app(TransactionJournalFactory::class);
$this->factory->setUser($job->user);
$config = $this->repository->getConfiguration($job);
$currency = app('amount')->getDefaultCurrencyByUser($job->user);
$this->defaultCurrencyId = $currency->id;
$this->job = $job;
$this->transfers = $this->getTransfers();
$this->applyRules = $config['apply-rules'] ?? false;
if (true === $this->applyRules) {
Log::debug('applyRules seems to be true, get the rules.');
$this->rules = $this->getRules();
}
Log::debug(sprintf('Value of apply rules is %s', var_export($this->applyRules, true)));
}
/**
* @param Collection $objects
*/
public function setObjects(Collection $objects)
{
$this->objects = $objects;
$this->total = $objects->count();
}
/**
* Do storage of import objects. Is the main function.
*
* @return bool
*/
public function store(): bool
{
$this->objects->each(
function (ImportJournal $importJournal, int $index) {
try {
$this->storeImportJournal($index, $importJournal);
$this->addStep();
} catch (Exception $e) {
$this->errors->push($e->getMessage());
Log::error(sprintf('Cannot import row #%d because: %s', $index, $e->getMessage()));
Log::error($e->getTraceAsString());
}
}
);
Log::info('ImportStorage has finished.');
return true;
}
/**
* @param int $index
* @param ImportJournal $importJournal
*
* @return bool
*
* @throws FireflyException
*/
protected function storeImportJournal(int $index, ImportJournal $importJournal): bool
{
Log::debug(sprintf('Going to store object #%d/%d with description "%s"', $index + 1, $this->total, $importJournal->getDescription()));
$assetAccount = $importJournal->asset->getAccount();
$amount = $importJournal->getAmount();
$foreignAmount = $importJournal->getForeignAmount();
$currencyId = $this->getCurrencyId($importJournal);
$foreignCurrencyId = $this->getForeignCurrencyId($importJournal, $currencyId);
$date = $importJournal->getDate($this->dateFormat)->format('Y-m-d');
$opposingAccount = $this->getOpposingAccount($importJournal->opposing, $assetAccount->id, $amount);
$transactionType = $this->getTransactionType($amount, $opposingAccount);
$this->addStep();
/**
* Check for double transfer.
*/
$parameters = [
'type' => $transactionType,
'description' => $importJournal->getDescription(),
'amount' => $amount,
'date' => $date,
'asset' => $assetAccount->name,
'opposing' => $opposingAccount->name,
];
if ($this->isDoubleTransfer($parameters) || $this->hashAlreadyImported($importJournal->hash)) {
// throw error
$message = sprintf('Detected a possible duplicate, skip this one (hash: %s).', $importJournal->hash);
Log::error($message, $parameters);
// add five steps to keep the pace:
$this->addSteps(5);
throw new FireflyException($message);
}
/**
* Search for journals with the same external ID.
*
*/
unset($parameters);
$this->addStep();
$budget = $importJournal->budget->getBudget();
$category = $importJournal->category->getCategory();
$bill = $importJournal->bill->getBill();
$source = $assetAccount;
$destination = $opposingAccount;
// switch account arounds when the transaction type is a deposit.
if ($transactionType === TransactionType::DEPOSIT) {
[$destination, $source] = [$source, $destination];
}
// switch accounts around when the amount is negative and it's a transfer.
// credits to @NyKoF
if ($transactionType === TransactionType::TRANSFER && -1 === bccomp($amount, '0')) {
[$destination, $source] = [$source, $destination];
}
Log::debug(
sprintf('Will make #%s (%s) the source and #%s (%s) the destination.', $source->id, $source->name, $destination->id, $destination->name)
);
$data = [
'user' => $this->job->user_id,
'type' => $transactionType,
'date' => $importJournal->getDate($this->dateFormat),
'description' => $importJournal->getDescription(),
'piggy_bank_id' => null,
'piggy_bank_name' => null,
'bill_id' => null === $bill ? null : $bill->id,
'bill_name' => null,
'tags' => $importJournal->tags,
'interest_date' => $importJournal->getMetaDate('interest_date'),
'book_date' => $importJournal->getMetaDate('book_date'),
'process_date' => $importJournal->getMetaDate('process_date'),
'due_date' => $importJournal->getMetaDate('due_date'),
'payment_date' => $importJournal->getMetaDate('payment_date'),
'invoice_date' => $importJournal->getMetaDate('invoice_date'),
'internal_reference' => $importJournal->metaFields['internal_reference'] ?? null,
'notes' => $importJournal->notes,
'external_id' => $importJournal->getExternalId(),
'sepa-cc' => $importJournal->getMetaString('sepa-cc'),
'sepa-ct-op' => $importJournal->getMetaString('sepa-ct-op'),
'sepa-ct-id' => $importJournal->getMetaString('sepa-ct-id'),
'sepa-db' => $importJournal->getMetaString('sepa-db'),
'sepa-country' => $importJournal->getMetaString('sepa-country'),
'sepa-ep' => $importJournal->getMetaString('sepa-ep'),
'sepa-ci' => $importJournal->getMetaString('sepa-ci'),
'importHash' => $importJournal->hash,
'transactions' => [
// single transaction:
[
'description' => null,
'amount' => $amount,
'currency_id' => $currencyId,
'currency_code' => null,
'foreign_amount' => $foreignAmount,
'foreign_currency_id' => $foreignCurrencyId,
'foreign_currency_code' => null,
'budget_id' => null === $budget ? null : $budget->id,
'budget_name' => null,
'category_id' => null === $category ? null : $category->id,
'category_name' => null,
'source_id' => $source->id,
'source_name' => null,
'destination_id' => $destination->id,
'destination_name' => null,
'reconciled' => false,
'identifier' => 0,
],
],
];
$factoryJournal = null;
try {
$factoryJournal = $this->factory->create($data);
$this->journals->push($factoryJournal);
} catch (FireflyException $e) {
Log::error(sprintf('Could not use factory to store journal: %s', $e->getMessage()));
Log::error($e->getTraceAsString());
}
// double add step because "match bills" no longer happens.
$this->addStep();
$this->addStep();
// run rules if config calls for it:
if (true === $this->applyRules && null !== $factoryJournal) {
Log::info('Will apply rules to this journal.');
$this->applyRules($factoryJournal);
}
Preferences::setForUser($this->job->user, 'lastActivity', microtime());
if (!(true === $this->applyRules)) {
Log::info('Will NOT apply rules to this journal.');
}
// double add step because some other extra thing was removed here.
$this->addStep();
$this->addStep();
Log::info(
sprintf(
'Imported new journal #%d: "%s", amount %s %s.', $factoryJournal->id, $factoryJournal->description, $factoryJournal->transactionCurrency->code,
$amount
)
);
return true;
}
/**
* Shorthand method.
*/
private function addStep()
{
$this->repository->addStepsDone($this->job, 1);
}
/**
* Shorthand method
*
* @param int $steps
*/
private function addSteps(int $steps)
{
$this->repository->addStepsDone($this->job, $steps);
}
/**
* @param array $parameters
*
* @return bool
*/
private function isDoubleTransfer(array $parameters): bool
{
Log::debug('Check if is a double transfer.');
if (TransactionType::TRANSFER !== $parameters['type']) {
Log::debug(sprintf('Is a %s, not a transfer so no.', $parameters['type']));
return false;
}
$amount = app('steam')->positive($parameters['amount']);
$names = [$parameters['asset'], $parameters['opposing']];
sort($names);
foreach ($this->transfers as $transfer) {
$hits = 0;
if ($parameters['description'] === $transfer['description']) {
++$hits;
Log::debug(sprintf('Description "%s" equals "%s", hits = %d', $parameters['description'], $transfer['description'], $hits));
}
if ($names === $transfer['names']) {
++$hits;
Log::debug(sprintf('Involved accounts, "%s" equals "%s", hits = %d', implode(',', $names), implode(',', $transfer['names']), $hits));
}
if (0 === bccomp($amount, $transfer['amount'])) {
++$hits;
Log::debug(sprintf('Amount %s equals %s, hits = %d', $amount, $transfer['amount'], $hits));
}
if ($parameters['date'] === $transfer['date']) {
++$hits;
Log::debug(sprintf('Date %s equals %s, hits = %d', $parameters['date'], $transfer['date'], $hits));
}
// number of hits is 4? Then it's a match
if (4 === $hits) {
Log::error(
'There already is a transfer imported with these properties. Compare existing with new. ',
['existing' => $transfer, 'new' => $parameters]
);
return true;
}
}
return false;
}
}

View File

@ -1,306 +0,0 @@
<?php
/**
* ImportSupport.php
* Copyright (c) 2017 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Import\Storage;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\Object\ImportAccount;
use FireflyIII\Import\Object\ImportJournal;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Models\Bill;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Rule;
use FireflyIII\Models\TransactionJournal;
use FireflyIII\Models\TransactionJournalMeta;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\Journal\JournalRepositoryInterface;
use FireflyIII\TransactionRules\Processor;
use Illuminate\Database\Query\JoinClause;
use Illuminate\Support\Collection;
use Log;
/**
* @codeCoverageIgnore
* @deprecated
* Trait ImportSupport.
*
* @property int $defaultCurrencyId
* @property ImportJob $job
* @property JournalRepositoryInterface $journalRepository;
* @property Collection $rules
*/
trait ImportSupport
{
/**
* @param TransactionJournal $journal
*
* @return bool
*/
protected function applyRules(TransactionJournal $journal): bool
{
if ($this->rules->count() > 0) {
$this->rules->each(
function (Rule $rule) use ($journal) {
Log::debug(sprintf('Going to apply rule #%d to journal %d.', $rule->id, $journal->id));
$processor = Processor::make($rule);
$processor->handleTransactionJournal($journal);
if ($rule->stop_processing) {
return false;
}
return true;
}
);
}
return true;
}
/**
* This method finds out what the import journal's currency should be. The account itself
* is favoured (and usually it stops there). If no preference is found, the journal has a say
* and thirdly the default currency is used.
*
* @param ImportJournal $importJournal
*
* @return int
*
* @throws FireflyException
*/
private function getCurrencyId(ImportJournal $importJournal): int
{
// start with currency pref of account, if any:
$account = $importJournal->asset->getAccount();
$currencyId = (int)$account->getMeta('currency_id');
if ($currencyId > 0) {
return $currencyId;
}
// use given currency
$currency = $importJournal->currency->getTransactionCurrency();
if (null !== $currency) {
return $currency->id;
}
// backup to default
$currency = $this->defaultCurrencyId;
return $currency;
}
/**
* The foreign currency is only returned when the journal has a different value from the
* currency id (see other method).
*
* @param ImportJournal $importJournal
* @param int $currencyId
*
* @see ImportSupport::getCurrencyId
*
* @return int|null
*/
private function getForeignCurrencyId(ImportJournal $importJournal, int $currencyId): ?int
{
// use given currency by import journal.
$currency = $importJournal->foreignCurrency->getTransactionCurrency();
if (null !== $currency && (int)$currency->id !== (int)$currencyId) {
return $currency->id;
}
// return null, because no different:
return null;
}
/**
* The search for the opposing account is complex. Firstly, we forbid the ImportAccount to resolve into the asset
* account to prevent a situation where the transaction flows from A to A. Given the amount, we "expect" the opposing
* account to be an expense or a revenue account. However, the mapping given by the user may return something else
* entirely (usually an asset account). So whatever the expectation, the result may be anything.
*
* When the result does not match the expected type (a negative amount cannot be linked to a revenue account) the next step
* will return an error.
*
* @param ImportAccount $account
* @param int $forbiddenAccount
* @param string $amount
*
* @see ImportSupport::getTransactionType
*
* @return Account
*
* @throws FireflyException
*/
private function getOpposingAccount(ImportAccount $account, int $forbiddenAccount, string $amount): Account
{
$account->setForbiddenAccountId($forbiddenAccount);
if (bccomp($amount, '0') === -1) {
Log::debug(sprintf('%s is negative, create opposing expense account.', $amount));
$account->setExpectedType(AccountType::EXPENSE);
return $account->getAccount();
}
Log::debug(sprintf('%s is positive, create opposing revenue account.', $amount));
// amount is positive, it's a deposit, opposing is an revenue:
$account->setExpectedType(AccountType::REVENUE);
return $account->getAccount();
}
/**
* @return Collection
*/
private function getRules(): Collection
{
$set = Rule::distinct()
->where('rules.user_id', $this->job->user->id)
->leftJoin('rule_groups', 'rule_groups.id', '=', 'rules.rule_group_id')
->leftJoin('rule_triggers', 'rules.id', '=', 'rule_triggers.rule_id')
->where('rule_groups.active', 1)
->where('rule_triggers.trigger_type', 'user_action')
->where('rule_triggers.trigger_value', 'store-journal')
->where('rules.active', 1)
->orderBy('rule_groups.order', 'ASC')
->orderBy('rules.order', 'ASC')
->get(['rules.*', 'rule_groups.order']);
Log::debug(sprintf('Found %d user rules.', $set->count()));
return $set;
}
/**
* Given the amount and the opposing account its easy to define which kind of transaction type should be associated with the new
* import. This may however fail when there is an unexpected mismatch between the transaction type and the opposing account.
*
* @param string $amount
* @param Account $account
*
* @return string
*x
*
* @throws FireflyException
*
* @see ImportSupport::getOpposingAccount()
*/
private function getTransactionType(string $amount, Account $account): string
{
$transactionType = TransactionType::WITHDRAWAL;
// amount is negative, it's a withdrawal, opposing is an expense:
if (bccomp($amount, '0') === -1) {
$transactionType = TransactionType::WITHDRAWAL;
}
if (1 === bccomp($amount, '0')) {
$transactionType = TransactionType::DEPOSIT;
}
// if opposing is an asset account, it's a transfer:
if (AccountType::ASSET === $account->accountType->type) {
Log::debug(sprintf('Opposing account #%d %s is an asset account, make transfer.', $account->id, $account->name));
$transactionType = TransactionType::TRANSFER;
}
// verify that opposing account is of the correct type:
if (AccountType::EXPENSE === $account->accountType->type && TransactionType::WITHDRAWAL !== $transactionType) {
$message = 'This row is imported as a withdrawal but opposing is an expense account. This cannot be!';
Log::error($message);
throw new FireflyException($message);
}
return $transactionType;
}
/**
* This method returns a collection of the current transfers in the system and some meta data for
* this set. This can later be used to see if the journal that firefly is trying to import
* is not already present.
*
* @return array
*/
private function getTransfers(): array
{
$set = TransactionJournal::leftJoin('transaction_types', 'transaction_types.id', '=', 'transaction_journals.transaction_type_id')
->leftJoin(
'transactions AS source',
function (JoinClause $join) {
$join->on('transaction_journals.id', '=', 'source.transaction_journal_id')->where('source.amount', '<', 0);
}
)
->leftJoin(
'transactions AS destination',
function (JoinClause $join) {
$join->on('transaction_journals.id', '=', 'destination.transaction_journal_id')->where(
'destination.amount',
'>',
0
);
}
)
->leftJoin('accounts as source_accounts', 'source.account_id', '=', 'source_accounts.id')
->leftJoin('accounts as destination_accounts', 'destination.account_id', '=', 'destination_accounts.id')
->where('transaction_journals.user_id', $this->job->user_id)
->where('transaction_types.type', TransactionType::TRANSFER)
->get(
['transaction_journals.id', 'transaction_journals.encrypted', 'transaction_journals.description',
'source_accounts.name as source_name', 'destination_accounts.name as destination_name', 'destination.amount',
'transaction_journals.date',]
);
$array = [];
/** @var TransactionJournal $entry */
foreach ($set as $entry) {
$original = [app('steam')->tryDecrypt($entry->source_name), app('steam')->tryDecrypt($entry->destination_name)];
sort($original);
$array[] = [
'names' => $original,
'amount' => $entry->amount,
'date' => $entry->date->format('Y-m-d'),
'description' => $entry->description,
];
}
return $array;
}
/**
* Checks if the import journal has not been imported before.
*
* @param string $hash
*
* @return bool
*/
private function hashAlreadyImported(string $hash): bool
{
$json = json_encode($hash);
/** @var TransactionJournalMeta $entry */
$entry = TransactionJournalMeta::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
->where('data', $json)
->where('name', 'importHash')
->first();
if (null !== $entry) {
Log::error(sprintf('A journal with hash %s has already been imported (spoiler: it\'s journal #%d)', $hash, $entry->transaction_journal_id));
return true;
}
return false;
}
}

View File

@ -24,15 +24,19 @@ declare(strict_types=1);
namespace FireflyIII\Support\Import\JobConfiguration\Bunq;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use Illuminate\Support\MessageBag;
use Log;
/**
* @codeCoverageIgnore
* Class NewBunqJobHandler
*/
class NewBunqJobHandler implements BunqJobConfigurationInterface
{
/** @var ImportJob */
private $importJob;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* Return true when this stage is complete.
@ -41,12 +45,16 @@ class NewBunqJobHandler implements BunqJobConfigurationInterface
*/
public function configurationComplete(): bool
{
Log::debug('NewBunqJobHandler::configurationComplete always returns true.');
// simply set the job configuration "apply-rules" to true.
$config = $this->repository->getConfiguration($this->importJob);
$config['apply-rules'] = true;
$this->repository->setConfiguration($this->importJob, $config);
return true;
}
/**
* @codeCoverageIgnore
* Store the job configuration.
*
* @param array $data
@ -61,6 +69,7 @@ class NewBunqJobHandler implements BunqJobConfigurationInterface
}
/**
* @codeCoverageIgnore
* Get data for config view.
*
* @return array
@ -73,6 +82,7 @@ class NewBunqJobHandler implements BunqJobConfigurationInterface
}
/**
* @codeCoverageIgnore
* Get the view for this stage.
*
* @return string
@ -91,6 +101,8 @@ class NewBunqJobHandler implements BunqJobConfigurationInterface
*/
public function setImportJob(ImportJob $importJob): void
{
Log::debug('NewBunqJobHandler::setImportJob does nothing.');
$this->importJob = $importJob;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($importJob->user);
}
}

View File

@ -0,0 +1,67 @@
<?php
/**
* NewBunqJobHandlerTest.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace Tests\Unit\Support\Import\JobConfiguration\Bunq;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\JobConfiguration\Bunq\NewBunqJobHandler;
use Mockery;
use Tests\TestCase;
class NewBunqJobHandlerTest extends TestCase
{
/**
* @covers \FireflyIII\Support\Import\JobConfiguration\Bunq\NewBunqJobHandler
*/
public function testCC(): void
{
$job = new ImportJob;
$job->user_id = $this->user()->id;
$job->key = 'cXha' . random_int(1, 1000);
$job->status = 'new';
$job->stage = 'new';
$job->provider = 'bunq';
$job->file_type = '';
$job->configuration = [];
$job->save();
// expected config:
$expected = [
'apply-rules' => true,
];
// mock stuff
$repository = $this->mock(ImportJobRepositoryInterface::class);
// mock calls
$repository->shouldReceive('setUser')->once();
$repository->shouldReceive('getConfiguration')->andReturn([])->once();
$repository->shouldReceive('setConfiguration')->withArgs([Mockery::any(), $expected])->once();
$handler = new NewBunqJobHandler();
$handler->setImportJob($job);
$this->assertTrue($handler->configurationComplete());
}
}