From cc1439fb7b591e56370d088f64d661f90a1f3f45 Mon Sep 17 00:00:00 2001 From: James Cole Date: Sat, 12 Aug 2017 19:03:42 +0200 Subject: [PATCH] Revamped import routine. --- app/Import/Object/ImportJournal.php | 49 +-- app/Import/Storage/ImportStorage.php | 585 ++++++--------------------- app/Import/Storage/ImportSupport.php | 421 +++++++++++++++++++ 3 files changed, 552 insertions(+), 503 deletions(-) create mode 100644 app/Import/Storage/ImportSupport.php diff --git a/app/Import/Object/ImportJournal.php b/app/Import/Object/ImportJournal.php index 6c5a5eb7e3..a72f574f28 100644 --- a/app/Import/Object/ImportJournal.php +++ b/app/Import/Object/ImportJournal.php @@ -17,9 +17,7 @@ use FireflyIII\Exceptions\FireflyException; use FireflyIII\Import\Converter\Amount; use FireflyIII\Import\Converter\ConverterInterface; use FireflyIII\Import\MapperPreProcess\PreProcessorInterface; -use FireflyIII\Models\TransactionJournal; use FireflyIII\User; -use Illuminate\Support\Collection; use InvalidArgumentException; use Log; use Steam; @@ -41,8 +39,6 @@ class ImportJournal public $category; /** @var string */ public $description = ''; - /** @var Collection */ - public $errors; /** @var string */ public $hash; /** @var array */ @@ -54,9 +50,9 @@ class ImportJournal /** @var array */ public $tags = []; /** @var string */ - private $amount = '0'; + private $amount; /** @var ImportCurrency */ - private $currency; + public $currency; /** @var string */ private $date = ''; /** @var string */ @@ -71,7 +67,6 @@ class ImportJournal */ public function __construct() { - $this->errors = new Collection; $this->asset = new ImportAccount; $this->opposing = new ImportAccount; $this->bill = new ImportBill; @@ -88,45 +83,29 @@ class ImportJournal $this->modifiers[] = $modifier; } - /** - * @return TransactionJournal - * @throws FireflyException - */ - public function createTransactionJournal(): TransactionJournal - { - exit('does not work yet'); - } - /** * @return string */ public function getAmount(): string { - - /** @var ConverterInterface $amountConverter */ - $amountConverter = app(Amount::class); - $this->amount = $amountConverter->convert($this->amount); - // modify - foreach ($this->modifiers as $modifier) { - $class = sprintf('FireflyIII\Import\Converter\%s', config(sprintf('csv.import_roles.%s.converter', $modifier['role']))); - /** @var ConverterInterface $converter */ - $converter = app($class); - if ($converter->convert($modifier['value']) === -1) { - $this->amount = Steam::negative($this->amount); + if (is_null($this->amount)) { + /** @var ConverterInterface $amountConverter */ + $amountConverter = app(Amount::class); + $this->amount = $amountConverter->convert($this->amount); + // modify + foreach ($this->modifiers as $modifier) { + $class = sprintf('FireflyIII\Import\Converter\%s', config(sprintf('csv.import_roles.%s.converter', $modifier['role']))); + /** @var ConverterInterface $converter */ + $converter = app($class); + if ($converter->convert($modifier['value']) === -1) { + $this->amount = Steam::negative($this->amount); + } } } return $this->amount; } - /** - * @return ImportCurrency - */ - public function getCurrency(): ImportCurrency - { - return $this->currency; - } - /** * @param string $format * diff --git a/app/Import/Storage/ImportStorage.php b/app/Import/Storage/ImportStorage.php index da915a5296..2f98ae6f46 100644 --- a/app/Import/Storage/ImportStorage.php +++ b/app/Import/Storage/ImportStorage.php @@ -12,12 +12,9 @@ declare(strict_types=1); namespace FireflyIII\Import\Storage; use FireflyIII\Exceptions\FireflyException; -use FireflyIII\Import\Object\ImportAccount; use FireflyIII\Import\Object\ImportJournal; -use FireflyIII\Models\Account; -use FireflyIII\Models\AccountType; use FireflyIII\Models\ImportJob; -use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface; +use FireflyIII\Models\TransactionType; use FireflyIII\Repositories\Tag\TagRepositoryInterface; use Illuminate\Support\Collection; use Log; @@ -30,16 +27,16 @@ use Log; */ class ImportStorage { + use ImportSupport; + /** @var Collection */ public $errors; /** @var Collection */ public $journals; - /** @var CurrencyRepositoryInterface */ - private $currencyRepository; - /** @var string */ - private $dateFormat = 'Ymd'; /** @var int */ - private $defaultCurrencyId = 0; + protected $defaultCurrencyId = 1; + /** @var string */ + private $dateFormat = 'Ymd'; // yes, hard coded /** @var ImportJob */ private $job; /** @var Collection */ @@ -49,6 +46,9 @@ class ImportStorage /** @var TagRepositoryInterface */ private $tagRepository; + /** @var array */ + private $transfers = []; + /** * ImportStorage constructor. */ @@ -75,15 +75,8 @@ class ImportStorage $this->job = $job; $currency = app('amount')->getDefaultCurrencyByUser($this->job->user); $this->defaultCurrencyId = $currency->id; - // $repository = app(CurrencyRepositoryInterface::class); - // $repository->setUser($job->user); - // $this->currencyRepository = $repository; - - // $repository = app(TagRepositoryInterface::class); - // $repository->setUser($job->user); - // $this->tagRepository = $repository; - // $this->rules = $this->getUserRules(); - // + $this->transfers = $this->getTransfers(); + $this->rules = $this->getRules(); } /** @@ -115,263 +108,6 @@ class ImportStorage return true; } - // - // /** - // * @param TransactionJournal $journal - // * - // * @return bool - // */ - // protected function applyRules(TransactionJournal $journal): bool - // { - // if ($this->rules->count() > 0) { - // /** @var Rule $rule */ - // foreach ($this->rules as $rule) { - // Log::debug(sprintf('Going to apply rule #%d to journal %d.', $rule->id, $journal->id)); - // $processor = Processor::make($rule); - // $processor->handleTransactionJournal($journal); - // - // if ($rule->stop_processing) { - // return true; - // } - // } - // } - // - // return true; - // } - - // /** - // * @param array $parameters - // * - // * @return bool - // * @throws FireflyException - // */ - // private function createTransaction(array $parameters): bool - // { - // $transaction = new Transaction; - // $transaction->account_id = $parameters['account']; - // $transaction->transaction_journal_id = $parameters['id']; - // $transaction->transaction_currency_id = $parameters['currency']; - // $transaction->amount = $parameters['amount']; - // $transaction->foreign_currency_id = $parameters['foreign_currency']; - // $transaction->foreign_amount = $parameters['foreign_amount']; - // $transaction->save(); - // if (is_null($transaction->id)) { - // $errorText = join(', ', $transaction->getErrors()->all()); - // throw new FireflyException($errorText); - // } - // Log::debug(sprintf('Created transaction with ID #%d, account #%d, amount %s', $transaction->id, $parameters['account'], $parameters['amount'])); - // - // return true; - // } - - // /** - // * @param Collection $set - // * @param ImportJournal $importJournal - // * - // * @return bool - // */ - // private function filterTransferSet(Collection $set, ImportJournal $importJournal): bool - // { - // $amount = Steam::positive($importJournal->getAmount()); - // $asset = $importJournal->asset->getAccount(); - // $opposing = $this->getOpposingAccount($importJournal->opposing, $asset->id, $amount); - // $description = $importJournal->getDescription(); - // - // $filtered = $set->filter( - // function (TransactionJournal $journal) use ($asset, $opposing, $description) { - // $match = true; - // $original = [app('steam')->tryDecrypt($journal->source_name), app('steam')->tryDecrypt($journal->destination_name)]; - // $compare = [$asset->name, $opposing->name]; - // sort($original); - // sort($compare); - // - // // description does not match? Then cannot be duplicate. - // if ($journal->description !== $description) { - // $match = false; - // } - // // not both accounts in journal? Then cannot be duplicate. - // if ($original !== $compare) { - // $match = false; - // } - // - // if ($match) { - // return $journal; - // } - // - // return null; - // } - // ); - // if (count($filtered) > 0) { - // return true; - // } - // - // return false; - // } - - // /** - // * @param ImportJournal $importJournal - // * - // * @param Account $account - // * - // * @return TransactionCurrency - // */ - // private function getCurrency(ImportJournal $importJournal, Account $account): TransactionCurrency - // { - // // start with currency pref of account, if any: - // $currency = $this->currencyRepository->find(intval($account->getMeta('currency_id'))); - // if (!is_null($currency->id)) { - // return $currency; - // } - // - // // use given currency - // $currency = $importJournal->getCurrency()->getTransactionCurrency(); - // if (!is_null($currency->id)) { - // return $currency; - // } - // - // // backup to default - // $currency = $this->defaultCurrency; - // - // return $currency; - // - // } - - // /** - // * @param ImportJournal $importJournal - // * @param TransactionCurrency $localCurrency - // * - // * @return int|null - // */ - // private function getForeignCurrencyId(ImportJournal $importJournal, TransactionCurrency $localCurrency): ?int - // { - // // get journal currency, if any: - // $currency = $importJournal->getCurrency()->getTransactionCurrency(); - // if (is_null($currency->id)) { - // Log::debug('getForeignCurrencyId: Journal has no currency, so can\'t be foreign either way.'); - // - // // journal has no currency, so can't be foreign either way: - // return null; - // } - // - // if ($currency->id !== $localCurrency->id) { - // Log::debug( - // sprintf('getForeignCurrencyId: journal is %s, but account is %s. Return id of journal currency.', $currency->code, $localCurrency->code) - // ); - // - // // journal has different currency than account does, return its ID: - // return $currency->id; - // } - // - // Log::debug('getForeignCurrencyId: journal has no foreign currency.'); - // - // // return null in other cases. - // return null; - // } - - /** - * @param ImportAccount $account - * @param $amount - * - * @return Account - */ - private function getOpposingAccount(ImportAccount $account, int $forbiddenAccount, string $amount): Account - { - $account->setForbiddenAccountId($forbiddenAccount); - if (bccomp($amount, '0') === -1) { - Log::debug(sprintf('%s is negative, create opposing expense account.', $amount)); - $account->setExpectedType(AccountType::EXPENSE); - - return $account->getAccount(); - } - Log::debug(sprintf('%s is positive, create opposing revenue account.', $amount)); - // amount is positive, it's a deposit, opposing is an revenue: - $account->setExpectedType(AccountType::REVENUE); - - $databaseAccount = $account->getAccount(); - - return $databaseAccount; - - } - // - // /** - // * @param string $amount - // * - // * @return TransactionType - // */ - // private function getTransactionType(string $amount): TransactionType - // { - // $transactionType = new TransactionType(); - // // amount is negative, it's a withdrawal, opposing is an expense: - // if (bccomp($amount, '0') === -1) { - // $transactionType = TransactionType::whereType(TransactionType::WITHDRAWAL)->first(); - // } - // if (bccomp($amount, '0') === 1) { - // $transactionType = TransactionType::whereType(TransactionType::DEPOSIT)->first(); - // } - // - // return $transactionType; - // } - - // /** - // * @return Collection - // */ - // private function getUserRules(): Collection - // { - // $set = Rule::distinct() - // ->where('rules.user_id', $this->job->user->id) - // ->leftJoin('rule_groups', 'rule_groups.id', '=', 'rules.rule_group_id') - // ->leftJoin('rule_triggers', 'rules.id', '=', 'rule_triggers.rule_id') - // ->where('rule_groups.active', 1) - // ->where('rule_triggers.trigger_type', 'user_action') - // ->where('rule_triggers.trigger_value', 'store-journal') - // ->where('rules.active', 1) - // ->orderBy('rule_groups.order', 'ASC') - // ->orderBy('rules.order', 'ASC') - // ->get(['rules.*', 'rule_groups.order']); - // Log::debug(sprintf('Found %d user rules.', $set->count())); - // - // return $set; - // - // } - // - // /** - // * @param TransactionJournal $journal - // * @param Bill $bill - // */ - // private function storeBill(TransactionJournal $journal, Bill $bill) - // { - // if (!is_null($bill->id)) { - // Log::debug(sprintf('Linked bill #%d to journal #%d', $bill->id, $journal->id)); - // $journal->bill()->associate($bill); - // $journal->save(); - // } - // } - - // /** - // * @param TransactionJournal $journal - // * @param Budget $budget - // */ - // private function storeBudget(TransactionJournal $journal, Budget $budget) - // { - // if (!is_null($budget->id)) { - // Log::debug(sprintf('Linked budget #%d to journal #%d', $budget->id, $journal->id)); - // $journal->budgets()->save($budget); - // } - // } - // - // /** - // * @param TransactionJournal $journal - // * @param Category $category - // */ - // private function storeCategory(TransactionJournal $journal, Category $category) - // { - // - // if (!is_null($category->id)) { - // Log::debug(sprintf('Linked category #%d to journal #%d', $category->id, $journal->id)); - // $journal->categories()->save($category); - // } - // - // } /** * @param int $index @@ -380,206 +116,119 @@ class ImportStorage * @return bool * @throws FireflyException */ - private function storeImportJournal(int $index, ImportJournal $importJournal): bool + protected function storeImportJournal(int $index, ImportJournal $importJournal): bool { Log::debug(sprintf('Going to store object #%d with description "%s"', $index, $importJournal->getDescription())); - $asset = $importJournal->asset->getAccount(); - $amount = $importJournal->getAmount(); + $assetAccount = $importJournal->asset->getAccount(); + $amount = $importJournal->getAmount(); + $currencyId = $this->getCurrencyId($importJournal); + $foreignCurrencyId = $this->getForeignCurrencyId($importJournal, $currencyId); + $date = $importJournal->getDate($this->dateFormat)->format('Y-m-d'); + $opposingAccount = $this->getOpposingAccount($importJournal->opposing, $assetAccount->id, $amount); + $transactionType = $this->getTransactionType($amount, $opposingAccount); + $description = $importJournal->getDescription(); + + /*** First step done! */ + $this->job->addStepsDone(1); + + /** + * Check for double transfer. + */ + $parameters = [ + 'type' => $transactionType, + 'description' => $description, + 'amount' => $amount, + 'date' => $date, + 'asset' => $assetAccount->name, + 'opposing' => $opposingAccount->name, + ]; + if ($this->isDoubleTransfer($parameters)) { + $this->job->addStepsDone(3); + // throw error + throw new FireflyException('Detected a possible duplicate, skip this one.'); + } + unset($parameters); + + // store journal and create transactions: + $parameters = [ + 'type' => $transactionType, + 'currency' => $currencyId, + 'foreign_currency' => $foreignCurrencyId, + 'asset' => $assetAccount, + 'opposing' => $opposingAccount, + 'description' => $description, + 'date' => $date, + 'hash' => $importJournal->hash, + 'amount' => $amount, + + ]; + $journal = $this->storeJournal($parameters); + unset($parameters); + + /*** Another step done! */ + $this->job->addStepsDone(1); + + // store meta object things: + $this->storeCategory($journal, $importJournal->category->getCategory()); + $this->storeBudget($journal, $importJournal->budget->getBudget()); + $this->storeBill($journal, $importJournal->bill->getBill()); + $this->storeMeta($journal, $importJournal->metaDates); + $journal->setMeta('notes', $importJournal->notes); + $this->storeTags($importJournal->tags, $journal); + + // set journal completed: + $journal->completed = true; + $journal->save(); + + /*** Another step done! */ + $this->job->addStepsDone(1); + + // run rules: + $this->applyRules($journal); + /*** Another step done! */ + $this->job->addStepsDone(1); + $this->journals->push($journal); + + Log::info( + sprintf( + 'Imported new journal #%d with description "%s" and amount %s %s.', $journal->id, $journal->description, $journal->transactionCurrency->code, + $amount + ) + ); - // $currency = $this->getCurrency($importJournal, $asset); - // $foreignCurrencyId = $this->getForeignCurrencyId($importJournal, $currency); - // $date = $importJournal->getDate($this->dateFormat); - // $transactionType = $this->getTransactionType($amount); - // $opposing = $this->getOpposingAccount($importJournal->opposing, $asset->id, $amount); - // - // // if opposing is an asset account, it's a transfer: - // if ($opposing->accountType->type === AccountType::ASSET) { - // Log::debug(sprintf('Opposing account #%d %s is an asset account, make transfer.', $opposing->id, $opposing->name)); - // $transactionType = TransactionType::whereType(TransactionType::TRANSFER)->first(); - // } - // - // // verify that opposing account is of the correct type: - // if ($opposing->accountType->type === AccountType::EXPENSE && $transactionType->type !== TransactionType::WITHDRAWAL) { - // $message = sprintf('Row #%d is imported as a %s but opposing is an expense account. This cannot be!', $index, $transactionType->type); - // Log::error($message); - // throw new FireflyException($message); - // - // } - // - // /*** First step done! */ - // $this->job->addStepsDone(1); - // - // // could be that transfer is double: verify this. - // if ($this->verifyDoubleTransfer($transactionType, $importJournal)) { - // // add three steps: - // $this->job->addStepsDone(3); - // // throw error - // throw new FireflyException('Detected a possible duplicate, skip this one.'); - // - // } - // - // // create a journal: - // $journal = new TransactionJournal; - // $journal->user_id = $this->job->user_id; - // $journal->transaction_type_id = $transactionType->id; - // $journal->transaction_currency_id = $currency->id;// always currency of account - // $journal->description = $importJournal->getDescription(); - // $journal->date = $date->format('Y-m-d'); - // $journal->order = 0; - // $journal->tag_count = 0; - // $journal->completed = false; - // - // if (!$journal->save()) { - // $errorText = join(', ', $journal->getErrors()->all()); - // // add three steps: - // $this->job->addStepsDone(3); - // // throw error - // throw new FireflyException($errorText); - // } - // - // // save meta data: - // $journal->setMeta('importHash', $importJournal->hash); - // Log::debug(sprintf('Created journal with ID #%d', $journal->id)); - // - // // create transactions: - // $one = [ - // 'id' => $journal->id, - // 'account' => $asset->id, - // 'currency' => $currency->id, - // 'amount' => $amount, - // 'foreign_currency' => $foreignCurrencyId, - // 'foreign_amount' => is_null($foreignCurrencyId) ? null : $amount, - // ]; - // $two = [ - // 'id' => $journal->id, - // 'account' => $opposing->id, - // 'currency' => $currency->id, - // 'amount' => Steam::opposite($amount), - // 'foreign_currency' => $foreignCurrencyId, - // 'foreign_amount' => is_null($foreignCurrencyId) ? null : Steam::opposite($amount), - // ]; - // $this->createTransaction($one); - // $this->createTransaction($two); - // - // /*** Another step done! */ - // $this->job->addStepsDone(1); - // - // // store meta object things: - // $this->storeCategory($journal, $importJournal->category->getCategory()); - // $this->storeBudget($journal, $importJournal->budget->getBudget()); - // $this->storeBill($journal, $importJournal->bill->getBill()); - // $this->storeMeta($journal, $importJournal->metaDates); - // - // // sepa thing as note: - // if (strlen($importJournal->notes) > 0) { - // $journal->setMeta('notes', $importJournal->notes); - // } - // - // // store tags - // $this->storeTags($importJournal->tags, $journal); - // - // // set journal completed: - // $journal->completed = true; - // $journal->save(); - // - // $this->job->addStepsDone(1); - // - // // run rules: - // $this->applyRules($journal); - // $this->job->addStepsDone(1); - // $this->journals->push($journal); - // - // Log::info( - // sprintf( - // 'Imported new journal #%d with description "%s" and amount %s %s.', $journal->id, $journal->description, $journal->transactionCurrency->code, - // $amount - // ) - // ); return true; } - // /** - // * @param TransactionJournal $journal - // * @param array $dates - // */ - // private function storeMeta(TransactionJournal $journal, array $dates) - // { - // // all other date fields as meta thing: - // foreach ($dates as $name => $value) { - // try { - // $date = new Carbon($value); - // $journal->setMeta($name, $date); - // } catch (\Exception $e) { - // // don't care, ignore: - // Log::warning(sprintf('Could not parse "%s" into a valid Date object for field %s', $value, $name)); - // } - // } - // } + /** + * @param array $parameters + * + * @return bool + */ + private function isDoubleTransfer(array $parameters): bool + { + if ($parameters['type'] !== TransactionType::TRANSFER) { + return false; + } - // /** - // * @param array $tags - // * @param TransactionJournal $journal - // */ - // private function storeTags(array $tags, TransactionJournal $journal): void - // { - // foreach ($tags as $tag) { - // $dbTag = $this->tagRepository->findByTag($tag); - // if (is_null($dbTag->id)) { - // $dbTag = $this->tagRepository->store( - // ['tag' => $tag, 'date' => null, 'description' => null, 'latitude' => null, 'longitude' => null, - // 'zoomLevel' => null, 'tagMode' => 'nothing'] - // ); - // } - // $journal->tags()->save($dbTag); - // Log::debug(sprintf('Linked tag %d ("%s") to journal #%d', $dbTag->id, $dbTag->tag, $journal->id)); - // } - // - // return; - // } + $amount = app('steam')->positive($parameters['amount']); + $names = [$parameters['asset'], $parameters['opposing']]; + sort($names); - // /** - // * This method checks if the given transaction is a transfer and if so, if it might be a duplicate of an already imported transfer. - // * This is important for import files that cover multiple accounts (and include both A<>B and B<>A transactions). - // * - // * @param TransactionType $transactionType - // * @param ImportJournal $importJournal - // * - // * @return bool - // */ - // private function verifyDoubleTransfer(TransactionType $transactionType, ImportJournal $importJournal): bool - // { - // if ($transactionType->type === TransactionType::TRANSFER) { - // $amount = Steam::positive($importJournal->getAmount()); - // $date = $importJournal->getDate($this->dateFormat); - // $set = TransactionJournal::leftJoin('transaction_types', 'transaction_types.id', '=', 'transaction_journals.transaction_type_id') - // ->leftJoin( - // 'transactions AS source', function (JoinClause $join) { - // $join->on('transaction_journals.id', '=', 'source.transaction_journal_id')->where('source.amount', '<', 0); - // } - // ) - // ->leftJoin( - // 'transactions AS destination', function (JoinClause $join) { - // $join->on('transaction_journals.id', '=', 'destination.transaction_journal_id')->where( - // 'destination.amount', '>', 0 - // ); - // } - // ) - // ->leftJoin('accounts as source_accounts', 'source.account_id', '=', 'source_accounts.id') - // ->leftJoin('accounts as destination_accounts', 'destination.account_id', '=', 'destination_accounts.id') - // ->where('transaction_journals.user_id', $this->job->user_id) - // ->where('transaction_types.type', TransactionType::TRANSFER) - // ->where('transaction_journals.date', $date->format('Y-m-d')) - // ->where('destination.amount', $amount) - // ->get( - // ['transaction_journals.id', 'transaction_journals.encrypted', 'transaction_journals.description', - // 'source_accounts.name as source_name', 'destination_accounts.name as destination_name'] - // ); - // - // return $this->filterTransferSet($set, $importJournal); - // } - // - // - // return false; - // } + foreach ($this->transfers as $transfer) { + if ($parameters['description'] !== $transfer['description']) { + return false; + } + if ($names !== $transfer['names']) { + return false; + } + if (bccomp($amount, $transfer['amount']) !== 0) { + return false; + } + if ($parameters['date'] !== $transfer['date']) { + return false; + } + } + return true; + } } diff --git a/app/Import/Storage/ImportSupport.php b/app/Import/Storage/ImportSupport.php new file mode 100644 index 0000000000..44ce15c4d9 --- /dev/null +++ b/app/Import/Storage/ImportSupport.php @@ -0,0 +1,421 @@ +rules->count() > 0) { + $this->rules->each( + function (Rule $rule) { + Log::debug(sprintf('Going to apply rule #%d to journal %d.', $rule->id, $journal->id)); + $processor = Processor::make($rule); + $processor->handleTransactionJournal($journal); + if ($rule->stop_processing) { + return false; + } + + return true; + } + ); + } + + return true; + } + + /** + * @param array $parameters + * + * @return bool + * @throws FireflyException + */ + private function createTransaction(array $parameters): bool + { + $transaction = new Transaction; + $transaction->account_id = $parameters['account']; + $transaction->transaction_journal_id = $parameters['id']; + $transaction->transaction_currency_id = $parameters['currency']; + $transaction->amount = $parameters['amount']; + $transaction->foreign_currency_id = $parameters['foreign_currency']; + $transaction->foreign_amount = $parameters['foreign_amount']; + $transaction->save(); + if (is_null($transaction->id)) { + $errorText = join(', ', $transaction->getErrors()->all()); + throw new FireflyException($errorText); + } + Log::debug(sprintf('Created transaction with ID #%d, account #%d, amount %s', $transaction->id, $parameters['account'], $parameters['amount'])); + + return true; + } + + /** + * This method finds out what the import journal's currency should be. The account itself + * is favoured (and usually it stops there). If no preference is found, the journal has a say + * and thirdly the default currency is used. + * + * @param ImportJournal $importJournal + * + * @return int + */ + private function getCurrencyId(ImportJournal $importJournal): int + { + // start with currency pref of account, if any: + $account = $importJournal->asset->getAccount(); + $currencyId = intval($account->getMeta('currency_id')); + if ($currencyId > 0) { + return $currencyId; + } + + // use given currency + $currency = $importJournal->currency->getTransactionCurrency(); + if (!is_null($currency->id)) { + return $currency->id; + } + + // backup to default + $currency = $this->defaultCurrencyId; + + return $currency; + + } + + /** + * The foreign currency is only returned when the journal has a different value from the + * currency id (see other method). + * + * @param ImportJournal $importJournal + * @param int $currencyId + * + * @see ImportSupport::getCurrencyId + * + * @return int|null + */ + private function getForeignCurrencyId(ImportJournal $importJournal, int $currencyId): ?int + { + // use given currency by import journal. + $currency = $importJournal->currency->getTransactionCurrency(); + if (!is_null($currency->id) && $currency->id !== $currencyId) { + return $currency->id; + } + + // return null, because no different: + return null; + } + + /** + * The search for the opposing account is complex. Firstly, we forbid the ImportAccount to resolve into the asset + * account to prevent a situation where the transaction flows from A to A. Given the amount, we "expect" the opposing + * account to be an expense or a revenue account. However, the mapping given by the user may return something else + * entirely (usually an asset account). So whatever the expectation, the result may be anything. + * + * When the result does not match the expected type (a negative amount cannot be linked to a revenue account) the next step + * will return an error. + * + * @param ImportAccount $account + * @param int $forbiddenAccount + * @param string $amount + * + * @see ImportSupport::getTransactionType + * + * @return Account + */ + private function getOpposingAccount(ImportAccount $account, int $forbiddenAccount, string $amount): Account + { + $account->setForbiddenAccountId($forbiddenAccount); + if (bccomp($amount, '0') === -1) { + Log::debug(sprintf('%s is negative, create opposing expense account.', $amount)); + $account->setExpectedType(AccountType::EXPENSE); + + return $account->getAccount(); + } + Log::debug(sprintf('%s is positive, create opposing revenue account.', $amount)); + // amount is positive, it's a deposit, opposing is an revenue: + $account->setExpectedType(AccountType::REVENUE); + + $databaseAccount = $account->getAccount(); + + return $databaseAccount; + + } + + /** + * @return Collection + */ + private function getRules(): Collection + { + $set = Rule::distinct() + ->where('rules.user_id', $this->job->user->id) + ->leftJoin('rule_groups', 'rule_groups.id', '=', 'rules.rule_group_id') + ->leftJoin('rule_triggers', 'rules.id', '=', 'rule_triggers.rule_id') + ->where('rule_groups.active', 1) + ->where('rule_triggers.trigger_type', 'user_action') + ->where('rule_triggers.trigger_value', 'store-journal') + ->where('rules.active', 1) + ->orderBy('rule_groups.order', 'ASC') + ->orderBy('rules.order', 'ASC') + ->get(['rules.*', 'rule_groups.order']); + Log::debug(sprintf('Found %d user rules.', $set->count())); + + return $set; + + } + + /** + * Given the amount and the opposing account its easy to define which kind of transaction type should be associated with the new + * import. This may however fail when there is an unexpected mismatch between the transaction type and the opposing account. + * + * @param string $amount + * @param Account $account + * + * @return string + * @throws FireflyException + * @see ImportSupport::getOpposingAccount() + */ + private function getTransactionType(string $amount, Account $account): string + { + $transactionType = ''; + // amount is negative, it's a withdrawal, opposing is an expense: + if (bccomp($amount, '0') === -1) { + $transactionType = TransactionType::WITHDRAWAL; + } + + if (bccomp($amount, '0') === 1) { + $transactionType = TransactionType::DEPOSIT; + } + + // if opposing is an asset account, it's a transfer: + if ($account->accountType->type === AccountType::ASSET) { + Log::debug(sprintf('Opposing account #%d %s is an asset account, make transfer.', $account->id, $opposing->name)); + $transactionType = TransactionType::whereType(TransactionType::TRANSFER)->first(); + } + + // verify that opposing account is of the correct type: + if ($account->accountType->type === AccountType::EXPENSE && $transactionType !== TransactionType::WITHDRAWAL) { + $message = sprintf('This row is imported as a %s but opposing is an expense account. This cannot be!', $transactionType); + Log::error($message); + throw new FireflyException($message); + } + + return $transactionType; + } + + /** + * This method returns a collection of the current transfers in the system and some meta data for + * this set. This can later be used to see if the journal that firefly is trying to import + * is not already present. + * + * @return array + */ + private function getTransfers(): array + { + $set = TransactionJournal::leftJoin('transaction_types', 'transaction_types.id', '=', 'transaction_journals.transaction_type_id') + ->leftJoin( + 'transactions AS source', function (JoinClause $join) { + $join->on('transaction_journals.id', '=', 'source.transaction_journal_id')->where('source.amount', '<', 0); + } + ) + ->leftJoin( + 'transactions AS destination', function (JoinClause $join) { + $join->on('transaction_journals.id', '=', 'destination.transaction_journal_id')->where( + 'destination.amount', '>', 0 + ); + } + ) + ->leftJoin('accounts as source_accounts', 'source.account_id', '=', 'source_accounts.id') + ->leftJoin('accounts as destination_accounts', 'destination.account_id', '=', 'destination_accounts.id') + ->where('transaction_journals.user_id', $this->job->user_id) + ->where('transaction_types.type', TransactionType::TRANSFER) + ->get( + ['transaction_journals.id', 'transaction_journals.encrypted', 'transaction_journals.description', + 'source_accounts.name as source_name', 'destination_accounts.name as destination_name', 'destination.amount' + , 'transaction_journals.date'] + ); + $array = []; + /** @var TransactionJournal $entry */ + foreach ($set as $entry) { + $original = [app('steam')->tryDecrypt($entry->source_name), app('steam')->tryDecrypt($entry->destination_name)]; + sort($original); + $array[] = [ + 'names' => $original, + 'amount' => $entry->amount, + 'date' => $entry->date->format('Y-m-d'), + 'description' => $entry->description, + ]; + } + + return $array; + } + + /** + * @param TransactionJournal $journal + * @param Bill $bill + */ + private function storeBill(TransactionJournal $journal, Bill $bill) + { + if (!is_null($bill->id)) { + Log::debug(sprintf('Linked bill #%d to journal #%d', $bill->id, $journal->id)); + $journal->bill()->associate($bill); + $journal->save(); + } + } + + /** + * @param TransactionJournal $journal + * @param Budget $budget + */ + private function storeBudget(TransactionJournal $journal, Budget $budget) + { + if (!is_null($budget->id)) { + Log::debug(sprintf('Linked budget #%d to journal #%d', $budget->id, $journal->id)); + $journal->budgets()->save($budget); + } + } + + /** + * @param TransactionJournal $journal + * @param Category $category + */ + private function storeCategory(TransactionJournal $journal, Category $category) + { + + if (!is_null($category->id)) { + Log::debug(sprintf('Linked category #%d to journal #%d', $category->id, $journal->id)); + $journal->categories()->save($category); + } + + } + + private function storeJournal(array $parameters): TransactionJournal + { + // find transaction type: + $transactionType = TransactionType::whereType($parameters['type'])->first(); + + // create a journal: + $journal = new TransactionJournal; + $journal->user_id = $this->job->user_id; + $journal->transaction_type_id = $transactionType->id; + $journal->transaction_currency_id = $parameters['currency']; + $journal->description = $parameters['description']; + $journal->date = $parameters['date']; + $journal->order = 0; + $journal->tag_count = 0; + $journal->completed = false; + + if (!$journal->save()) { + $errorText = join(', ', $journal->getErrors()->all()); + // add three steps: + $this->job->addStepsDone(3); + // throw error + throw new FireflyException($errorText); + } + // save meta data: + $journal->setMeta('importHash', $parameters['hash']); + Log::debug(sprintf('Created journal with ID #%d', $journal->id)); + + // create transactions: + $one = [ + 'id' => $journal->id, + 'account' => $parameters['asset']->id, + 'currency' => $parameters['currency'], + 'amount' => $parameters['amount'], + 'foreign_currency' => $parameters['foreign_currency'], + 'foreign_amount' => is_null($parameters['foreign_currency']) ? null : $parameters['amount'], + ]; + $opposite = app('steam')->opposite($parameters['amount']); + $two = [ + 'id' => $journal->id, + 'account' => $parameters['opposing']->id, + 'currency' => $parameters['currency'], + 'amount' => $opposite, + 'foreign_currency' => $parameters['foreign_currency'], + 'foreign_amount' => is_null($parameters['foreign_currency']) ? null : $opposite, + ]; + $this->createTransaction($one); + $this->createTransaction($two); + + return $journal; + } + + /** + * @param TransactionJournal $journal + * @param array $dates + */ + private function storeMeta(TransactionJournal $journal, array $dates) + { + // all other date fields as meta thing: + foreach ($dates as $name => $value) { + try { + $date = new Carbon($value); + $journal->setMeta($name, $date); + } catch (Exception $e) { + // don't care, ignore: + Log::warning(sprintf('Could not parse "%s" into a valid Date object for field %s', $value, $name)); + } + } + } + + /** + * @param array $tags + * @param TransactionJournal $journal + */ + private function storeTags(array $tags, TransactionJournal $journal): void + { + $repository = app(TagRepositoryInterface::class); + $repository->setUser($journal->user); + + foreach ($tags as $tag) { + $dbTag = $repository->findByTag($tag); + if (is_null($dbTag->id)) { + $dbTag = $repository->store( + ['tag' => $tag, 'date' => null, 'description' => null, 'latitude' => null, 'longitude' => null, + 'zoomLevel' => null, 'tagMode' => 'nothing'] + ); + } + $journal->tags()->save($dbTag); + Log::debug(sprintf('Linked tag %d ("%s") to journal #%d', $dbTag->id, $dbTag->tag, $journal->id)); + } + + return; + } + +} \ No newline at end of file