diff --git a/app/Repositories/Account/AccountRepositoryInterface.php b/app/Repositories/Account/AccountRepositoryInterface.php index 8c8bdaf7ea..7820888a99 100644 --- a/app/Repositories/Account/AccountRepositoryInterface.php +++ b/app/Repositories/Account/AccountRepositoryInterface.php @@ -66,10 +66,9 @@ interface AccountRepositoryInterface * @param string $number * @param array $types * - * @deprecated - * @return Account + * @return Account|null */ - public function findByAccountNumber(string $number, array $types): Account; + public function findByAccountNumber(string $number, array $types): ?Account; /** * @param string $iban diff --git a/app/Repositories/Account/FindAccountsTrait.php b/app/Repositories/Account/FindAccountsTrait.php index d520af06fe..79451ff6e4 100644 --- a/app/Repositories/Account/FindAccountsTrait.php +++ b/app/Repositories/Account/FindAccountsTrait.php @@ -58,12 +58,9 @@ trait FindAccountsTrait /** * @param string $number * @param array $types - * - * - * @deprecated - * @return Account + * @return Account|null */ - public function findByAccountNumber(string $number, array $types): Account + public function findByAccountNumber(string $number, array $types): ?Account { $query = $this->user->accounts() ->leftJoin('account_meta', 'account_meta.account_id', '=', 'accounts.id') @@ -81,7 +78,7 @@ trait FindAccountsTrait return $accounts->first(); } - return new Account; + return null; } /** diff --git a/app/Support/Import/Placeholder/ImportTransaction.php b/app/Support/Import/Placeholder/ImportTransaction.php index e928b7be2b..86b19eb2e9 100644 --- a/app/Support/Import/Placeholder/ImportTransaction.php +++ b/app/Support/Import/Placeholder/ImportTransaction.php @@ -24,6 +24,11 @@ declare(strict_types=1); namespace FireflyIII\Support\Import\Placeholder; use FireflyIII\Exceptions\FireflyException; +use FireflyIII\Import\Converter\Amount; +use FireflyIII\Import\Converter\AmountCredit; +use FireflyIII\Import\Converter\AmountDebit; +use FireflyIII\Import\Converter\ConverterInterface; +use Log; /** * Class ImportTransaction @@ -97,24 +102,6 @@ class ImportTransaction /** @var array */ private $tags; - /** - * @return array - */ - public function getMeta(): array - { - return $this->meta; - } - - /** - * @return string - */ - public function getNote(): string - { - return $this->note; - } - - - /** * ImportTransaction constructor. */ @@ -137,32 +124,6 @@ class ImportTransaction } - /** - * @return string - */ - public function getDescription(): string - { - return $this->description; - } - - /** - * @return int - */ - public function getBillId(): int - { - return $this->billId; - } - - /** - * @return null|string - */ - public function getBillName(): ?string - { - return $this->billName; - } - - - /** * @param ColumnValue $columnValue * @@ -305,6 +266,135 @@ class ImportTransaction } } + /** + * Calculate the amount of this transaction. + * + * @return string + * @throws FireflyException + */ + public function calculateAmount(): string + { + Log::debug('Now in importTransaction->calculateAmount()'); + $info = $this->selectAmountInput(); + + if (0 === \count($info)) { + throw new FireflyException('No amount information for this row.'); + } + $class = $info['class'] ?? ''; + if (0 === \strlen($class)) { + throw new FireflyException('No amount information (conversion class) for this row.'); + } + + Log::debug(sprintf('Converter class is %s', $info['class'])); + /** @var ConverterInterface $amountConverter */ + $amountConverter = app($info['class']); + $result = $amountConverter->convert($info['amount']); + Log::debug(sprintf('First attempt to convert gives "%s"', $result)); + // modify + /** + * @var string $role + * @var string $modifier + */ + foreach ($this->modifiers as $role => $modifier) { + $class = sprintf('FireflyIII\Import\Converter\%s', config(sprintf('csv.import_roles.%s.converter', $role))); + /** @var ConverterInterface $converter */ + $converter = app($class); + Log::debug(sprintf('Now launching converter %s', $class)); + $conversion = $converter->convert($modifier); + if ($conversion === -1) { + $result = app('steam')->negative($result); + } + if ($conversion === 1) { + $result = app('steam')->positive($result); + } + Log::debug(sprintf('convertedAmount after conversion is %s', $result)); + } + + Log::debug(sprintf('After modifiers the result is: "%s"', $result)); + + + return $result; + } + + /** + * This array is being used to map the account the user is using. + * + * @return array + */ + public function getAccountData(): array + { + return [ + 'iban' => $this->accountIban, + 'name' => $this->accountName, + 'number' => $this->accountNumber, + 'bic' => $this->accountBic, + ]; + } + + /** + * @return int + */ + public function getAccountId(): int + { + return $this->accountId; + } + + /** + * @return int + */ + public function getBillId(): int + { + return $this->billId; + } + + /** + * @return null|string + */ + public function getBillName(): ?string + { + return $this->billName; + } + + /** + * @return int + */ + public function getBudgetId(): int + { + return $this->budgetId; + } + + /** + * @return string|null + */ + public function getBudgetName(): ?string + { + return $this->budgetName; + } + + /** + * @return int + */ + public function getCategoryId(): int + { + return $this->categoryId; + } + + /** + * @return string|null + */ + public function getCategoryName(): ?string + { + return $this->categoryName; + } + + /** + * @return int + */ + public function getCurrencyId(): int + { + return $this->currencyId; + } + /** * @return string */ @@ -313,11 +403,64 @@ class ImportTransaction return $this->date; } + /** + * @return string + */ + public function getDescription(): string + { + return $this->description; + } + + /** + * @return int + */ + public function getForeignCurrencyId(): int + { + return $this->foreignCurrencyId; + } + + /** + * @return array + */ + public function getMeta(): array + { + return $this->meta; + } + + /** + * @return string + */ + public function getNote(): string + { + return $this->note; + } + + public function getOpposingAccountData(): array + { + return [ + 'iban' => $this->opposingIban, + 'name' => $this->opposingName, + 'number' => $this->opposingNumber, + 'bic' => $this->opposingBic, + ]; + } + + /** + * @return int + */ + public function getOpposingId(): int + { + return $this->opposingId; + } + /** * @return array */ public function getTags(): array { + return []; + + // todo make sure this is an array return $this->tags; } @@ -333,4 +476,33 @@ class ImportTransaction return $columnValue->getMappedValue() > 0 ? $columnValue->getMappedValue() : (int)$columnValue->getValue(); } + /** + * This methods decides which input value to use for the amount calculation. + * + * @return array + */ + private function selectAmountInput() + { + $info = []; + $converterClass = ''; + if (null !== $this->amount) { + Log::debug('Amount value is not NULL, assume this is the correct value.'); + $converterClass = Amount::class; + $info['amount'] = $this->amount; + } + if (null !== $this->amountDebit) { + Log::debug('Amount DEBIT value is not NULL, assume this is the correct value (overrules Amount).'); + $converterClass = AmountDebit::class; + $info['amount'] = $this->amountDebit; + } + if (null !== $this->amountCredit) { + Log::debug('Amount CREDIT value is not NULL, assume this is the correct value (overrules Amount and AmountDebit).'); + $converterClass = AmountCredit::class; + $info['amount'] = $this->amountCredit; + } + $info['class'] = $converterClass; + + return $info; + } + } \ No newline at end of file diff --git a/app/Support/Import/Routine/File/CSVProcessor.php b/app/Support/Import/Routine/File/CSVProcessor.php index 0698da05d6..1edfb5ddd8 100644 --- a/app/Support/Import/Routine/File/CSVProcessor.php +++ b/app/Support/Import/Routine/File/CSVProcessor.php @@ -26,6 +26,8 @@ namespace FireflyIII\Support\Import\Routine\File; use Carbon\Carbon; use FireflyIII\Exceptions\FireflyException; use FireflyIII\Helpers\Attachments\AttachmentHelperInterface; +use FireflyIII\Models\Account; +use FireflyIII\Models\AccountType; use FireflyIII\Models\Attachment; use FireflyIII\Models\ImportJob; use FireflyIII\Repositories\Account\AccountRepositoryInterface; @@ -50,6 +52,8 @@ use Log; */ class CSVProcessor implements FileProcessorInterface { + /** @var AccountRepositoryInterface */ + private $accountRepos; /** @var AttachmentHelperInterface */ private $attachments; /** @var array */ @@ -69,7 +73,7 @@ class CSVProcessor implements FileProcessorInterface */ public function run(): array { - + Log::debug('Now in CSVProcessor() run'); // in order to actually map we also need to read the FULL file. try { @@ -91,6 +95,7 @@ class CSVProcessor implements FileProcessorInterface $array = $this->parseImportables($importables); echo '
'; + print_r($array); print_r($importables); print_r($lines); @@ -103,11 +108,14 @@ class CSVProcessor implements FileProcessorInterface */ public function setJob(ImportJob $job): void { - $this->importJob = $job; - $this->config = $job->configuration; - $this->repository = app(ImportJobRepositoryInterface::class); - $this->attachments = app(AttachmentHelperInterface::class); + Log::debug('Now in setJob()'); + $this->importJob = $job; + $this->config = $job->configuration; + $this->repository = app(ImportJobRepositoryInterface::class); + $this->attachments = app(AttachmentHelperInterface::class); + $this->accountRepos = app(AccountRepositoryInterface::class); $this->repository->setUser($job->user); + $this->accountRepos->setUser($job->user); } @@ -121,6 +129,7 @@ class CSVProcessor implements FileProcessorInterface */ private function getLines(Reader $reader): array { + Log::debug('now in getLines()'); $offset = isset($this->config['has-headers']) && $this->config['has-headers'] === true ? 1 : 0; try { $stmt = (new Statement)->offset($offset); @@ -143,6 +152,7 @@ class CSVProcessor implements FileProcessorInterface */ private function getReader(): Reader { + Log::debug('Now in getReader()'); $content = ''; /** @var Collection $collection */ $collection = $this->importJob->attachments; @@ -195,28 +205,28 @@ class CSVProcessor implements FileProcessorInterface case 'account-number': $newRole = 'account-id'; break; - case 'foreign-currency-id': - case 'foreign-currency-code': - $newRole = 'foreign-currency-id'; - break; case 'bill-id': case 'bill-name': $newRole = 'bill-id'; break; + case 'budget-id': + case 'budget-name': + $newRole = 'budget-id'; + break; case 'currency-id': case 'currency-name': case 'currency-code': case 'currency-symbol': $newRole = 'currency-id'; break; - case 'budget-id': - case 'budget-name': - $newRole = 'budget-id'; - break; case 'category-id': case 'category-name': $newRole = 'category-id'; break; + case 'foreign-currency-id': + case 'foreign-currency-code': + $newRole = 'foreign-currency-id'; + break; case 'opposing-id': case 'opposing-name': case 'opposing-iban': @@ -232,6 +242,120 @@ class CSVProcessor implements FileProcessorInterface return $newRole; } + /** + * Based upon data in the importable, try to find or create the asset account account. + * + * @param $importable + * + * @return Account + */ + private function mapAssetAccount(?int $accountId, array $accountData): Account + { + Log::debug('Now in mapAssetAccount()'); + if ((int)$accountId > 0) { + // find asset account with this ID: + $result = $this->accountRepos->findNull($accountId); + if (null !== $result) { + Log::debug(sprintf('Found account "%s" based on given ID %d. Return it!', $result->name, $accountId)); + + return $result; + } + } + // find by (respectively): + // IBAN, accountNumber, name, + $fields = ['iban' => 'findByIbanNull', 'number' => 'findByAccountNumber', 'name' => 'findByName']; + foreach ($fields as $field => $function) { + $value = $accountData[$field]; + if (null === $value) { + continue; + } + $result = $this->accountRepos->$function($value, [AccountType::ASSET]); + Log::debug(sprintf('Going to run %s() with argument "%s" (asset account)', $function, $value)); + if (null !== $result) { + Log::debug(sprintf('Found asset account "%s". Return it!', $result->name, $accountId)); + + return $result; + } + } + Log::debug('Found nothing. Will return default account.'); + // still NULL? Return default account. + $default = null; + if (isset($this->config['import-account'])) { + $default = $this->accountRepos->findNull((int)$this->config['import-account']); + } + if (null === $default) { + Log::debug('Default account is NULL! Simply result first account in system.'); + $default = $this->accountRepos->getAccountsByType([AccountType::ASSET])->first(); + } + + Log::debug(sprintf('Return default account "%s" (#%d). Return it!', $default->name, $default->id)); + + return $default; + } + + /** + * @param int|null $accountId + * @param string $amount + * @param array $accountData + * + * @return Account + */ + private function mapOpposingAccount(?int $accountId, string $amount, array $accountData): Account + { + Log::debug('Now in mapOpposingAccount()'); + if ((int)$accountId > 0) { + // find any account with this ID: + $result = $this->accountRepos->findNull($accountId); + if (null !== $result) { + Log::debug(sprintf('Found account "%s" (%s) based on given ID %d. Return it!', $result->name, $result->accountType->type, $accountId)); + + return $result; + } + } + // default assumption is we're looking for an expense account. + $expectedType = AccountType::EXPENSE; + Log::debug(sprintf('Going to search for accounts of type %s', $expectedType)); + if (bccomp($amount, '0') === 1) { + // more than zero. + $expectedType = AccountType::REVENUE; + Log::debug(sprintf('Because amount is %s, will instead search for accounts of type %s', $amount, $expectedType)); + } + + // first search for $expectedType, then find asset: + $searchTypes = [$expectedType, AccountType::ASSET]; + foreach ($searchTypes as $type) { + // find by (respectively): + // IBAN, accountNumber, name, + $fields = ['iban' => 'findByIbanNull', 'number' => 'findByAccountNumber', 'name' => 'findByName']; + foreach ($fields as $field => $function) { + $value = $accountData[$field]; + if (null === $value) { + continue; + } + Log::debug(sprintf('Will search for account of type "%s" using %s() and argument %s.', $type, $function, $value)); + $result = $this->accountRepos->$function($value, [$type]); + if (null !== $result) { + Log::debug(sprintf('Found result: Account #%d, named "%s"', $result->id, $result->name)); + + return $result; + } + } + } + // not found? Create it! + $creation = [ + 'name' => $accountData['name'], + 'iban' => $accountData['iban'], + 'accountNumber' => $accountData['number'], + 'account_type_id' => null, + 'accountType' => $expectedType, + 'active' => true, + 'BIC' => $accountData['bic'], + ]; + Log::debug('Will try to store a new account: ', $creation); + + return $this->accountRepos->store($creation); + } + /** * Each entry is an ImportTransaction that must be converted to an array compatible with the * journal factory. To do so some stuff must still be resolved. See below. @@ -239,76 +363,119 @@ class CSVProcessor implements FileProcessorInterface * @param array $importables * * @return array + * @throws FireflyException */ private function parseImportables(array $importables): array { + Log::debug('Now in parseImportables()'); $array = []; + $total = count($importables); /** @var ImportTransaction $importable */ - foreach ($importables as $importable) { - - // todo: verify bill mapping - // todo: verify currency mapping. - - - $entry = [ - 'type' => 'unknown', // todo - 'date' => Carbon::createFromFormat($this->config['date-format'] ?? 'Ymd', $importable->getDate()), - 'tags' => $importable->getTags(), // todo make sure its filled. - 'user' => $this->importJob->user_id, - 'notes' => $importable->getNote(), - - // all custom fields: - 'internal_reference' => $importable->getMeta()['internal-reference'] ?? null, - 'sepa-cc' => $importable->getMeta()['sepa-cc'] ?? null, - 'sepa-ct-op' => $importable->getMeta()['sepa-ct-op'] ?? null, - 'sepa-ct-id' => $importable->getMeta()['sepa-ct-id'] ?? null, - 'sepa-db' => $importable->getMeta()['sepa-db'] ?? null, - 'sepa-country' => $importable->getMeta()['sepa-countru'] ?? null, - 'sepa-ep' => $importable->getMeta()['sepa-ep'] ?? null, - 'sepa-ci' => $importable->getMeta()['sepa-ci'] ?? null, - 'interest_date' => $importable->getMeta()['date-interest'] ?? null, - 'book_date' => $importable->getMeta()['date-book'] ?? null, - 'process_date' => $importable->getMeta()['date-process'] ?? null, - 'due_date' => $importable->getMeta()['date-due'] ?? null, - 'payment_date' => $importable->getMeta()['date-payment'] ?? null, - 'invoice_date' => $importable->getMeta()['date-invoice'] ?? null, - // todo external ID - - // journal data: - 'description' => $importable->getDescription(), - 'piggy_bank_id' => null, - 'piggy_bank_name' => null, - 'bill_id' => $importable->getBillId() === 0 ? null : $importable->getBillId(), // - 'bill_name' => $importable->getBillId() !== 0 ? null : $importable->getBillName(), - - // transaction data: - 'transactions' => [ - [ - 'currency_id' => null, // todo find ma - 'currency_code' => 'EUR', - 'description' => null, - 'amount' => random_int(500, 5000) / 100, - 'budget_id' => null, - 'budget_name' => null, - 'category_id' => null, - 'category_name' => null, - 'source_id' => null, - 'source_name' => 'Checking Account', - 'destination_id' => null, - 'destination_name' => 'Random expense account #' . random_int(1, 10000), - 'foreign_currency_id' => null, - 'foreign_currency_code' => null, - 'foreign_amount' => null, - 'reconciled' => false, - 'identifier' => 0, - ], - ], - ]; + foreach ($importables as $index => $importable) { + Log::debug(sprintf('Now going to parse importable %d of %d', $index + 1, $total)); + $array[] = $this->parseSingleImportable($importable); } return $array; } + /** + * @param ImportTransaction $importable + * + * @return array + * @throws FireflyException + */ + private function parseSingleImportable(ImportTransaction $importable): array + { + + $amount = $importable->calculateAmount(); + + /** + * first finalise the amount. cehck debit en credit. + * then get the accounts. + * ->account always assumes were looking for an asset account. + * cannot create anything, will return the default account when nothing comes up. + * + * neg + account = assume asset account? + * neg = assume withdrawal + * pos = assume + */ + + $accountId = $this->verifyObjectId('account-id', $importable->getAccountId()); + $billId = $this->verifyObjectId('bill-id', $importable->getForeignCurrencyId()); + $budgetId = $this->verifyObjectId('budget-id', $importable->getBudgetId()); + $currencyId = $this->verifyObjectId('currency-id', $importable->getForeignCurrencyId()); + $categoryId = $this->verifyObjectId('category-id', $importable->getCategoryId()); + $foreignCurrencyId = $this->verifyObjectId('foreign-currency-id', $importable->getForeignCurrencyId()); + $opposingId = $this->verifyObjectId('opposing-id', $importable->getOpposingId()); + // also needs amount to be final. + //$account = $this->mapAccount($accountId, $importable->getAccountData()); + $asset = $this->mapAssetAccount($accountId, $importable->getAccountData()); + $sourceId = $asset->id; + $opposing = $this->mapOpposingAccount($opposingId, $amount, $importable->getOpposingAccountData()); + $destId = $opposing->id; + + if (bccomp($amount, '0') === 1) { + // amount is positive? Then switch: + [$destId, $sourceId] = [$sourceId, $destId]; + } + + return [ + 'type' => 'unknown', // todo + 'date' => Carbon::createFromFormat($this->config['date-format'] ?? 'Ymd', $importable->getDate()), + 'tags' => $importable->getTags(), // todo make sure its filled. + 'user' => $this->importJob->user_id, + 'notes' => $importable->getNote(), + + // all custom fields: + 'internal_reference' => $importable->getMeta()['internal-reference'] ?? null, + 'sepa-cc' => $importable->getMeta()['sepa-cc'] ?? null, + 'sepa-ct-op' => $importable->getMeta()['sepa-ct-op'] ?? null, + 'sepa-ct-id' => $importable->getMeta()['sepa-ct-id'] ?? null, + 'sepa-db' => $importable->getMeta()['sepa-db'] ?? null, + 'sepa-country' => $importable->getMeta()['sepa-countru'] ?? null, + 'sepa-ep' => $importable->getMeta()['sepa-ep'] ?? null, + 'sepa-ci' => $importable->getMeta()['sepa-ci'] ?? null, + 'interest_date' => $importable->getMeta()['date-interest'] ?? null, + 'book_date' => $importable->getMeta()['date-book'] ?? null, + 'process_date' => $importable->getMeta()['date-process'] ?? null, + 'due_date' => $importable->getMeta()['date-due'] ?? null, + 'payment_date' => $importable->getMeta()['date-payment'] ?? null, + 'invoice_date' => $importable->getMeta()['date-invoice'] ?? null, + // todo external ID + + // journal data: + 'description' => $importable->getDescription(), + 'piggy_bank_id' => null, + 'piggy_bank_name' => null, + 'bill_id' => $billId, + 'bill_name' => null === $budgetId ? $importable->getBillName() : null, + + // transaction data: + 'transactions' => [ + [ + 'currency_id' => $currencyId, // todo what if null? + 'currency_code' => null, + 'description' => $importable->getDescription(), + 'amount' => $amount, + 'budget_id' => $budgetId, + 'budget_name' => null === $budgetId ? $importable->getBudgetName() : null, + 'category_id' => $categoryId, + 'category_name' => null === $categoryId ? $importable->getCategoryName() : null, + 'source_id' => $sourceId, + 'source_name' => null, + 'destination_id' => $destId, + 'destination_name' => null, + 'foreign_currency_id' => $foreignCurrencyId, + 'foreign_currency_code' => null, + 'foreign_amount' => null, // todo get me. + 'reconciled' => false, + 'identifier' => 0, + ], + ], + ]; + } + /** * Process all lines in the CSV file. Each line is processed separately. * @@ -319,6 +486,7 @@ class CSVProcessor implements FileProcessorInterface */ private function processLines(array $lines): array { + Log::debug('Now in processLines()'); $processed = []; $count = \count($lines); foreach ($lines as $index => $line) { @@ -341,11 +509,14 @@ class CSVProcessor implements FileProcessorInterface */ private function processSingleLine(array $line): ImportTransaction { + Log::debug('Now in processSingleLine()'); $transaction = new ImportTransaction; // todo run all specifics on row. foreach ($line as $column => $value) { + $value = trim($value); $originalRole = $this->config['column-roles'][$column] ?? '_ignore'; + Log::debug(sprintf('Now at column #%d (%s), value "%s"', $column, $originalRole, $value)); if (\strlen($value) > 0 && $originalRole !== '_ignore') { // is a mapped value present? @@ -359,8 +530,9 @@ class CSVProcessor implements FileProcessorInterface $columnValue->setMappedValue($mapped); $columnValue->setOriginalRole($originalRole); $transaction->addColumnValue($columnValue); - - Log::debug(sprintf('Now at column #%d (%s), value "%s"', $column, $role, $value)); + } + if ('' === $value) { + Log::debug('Column skipped because value is empty.'); } } @@ -377,6 +549,7 @@ class CSVProcessor implements FileProcessorInterface */ private function validateMappedValues() { + Log::debug('Now in validateMappedValues()'); foreach ($this->mappedValues as $role => $values) { $values = array_unique($values); if (count($values) > 0) { @@ -385,10 +558,7 @@ class CSVProcessor implements FileProcessorInterface throw new FireflyException(sprintf('Cannot validate mapped values for role "%s"', $role)); case 'opposing-id': case 'account-id': - /** @var AccountRepositoryInterface $repository */ - $repository = app(AccountRepositoryInterface::class); - $repository->setUser($this->importJob->user); - $set = $repository->getAccountsById($values); + $set = $this->accountRepos->getAccountsById($values); $valid = $set->pluck('id')->toArray(); $this->mappedValues[$role] = $valid; break; @@ -429,4 +599,22 @@ class CSVProcessor implements FileProcessorInterface } } } + + /** + * A small function that verifies if this particular key (ID) is present in the list + * of valid keys. + * + * @param string $key + * @param int $objectId + * + * @return int|null + */ + private function verifyObjectId(string $key, int $objectId): ?int + { + if (isset($this->mappedValues[$key]) && in_array($objectId, $this->mappedValues[$key])) { + return $objectId; + } + + return null; + } } \ No newline at end of file