IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2024-12-27 09:21:04 -06:00
Code Issues Packages Projects Releases Wiki Activity

Improve importer.

Browse Source
This commit is contained in:
James Cole 2018-05-09 20:53:39 +02:00
parent 7f4feb0cfc
commit 6ef0eb73d0
4 changed files with 488 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/Repositories/Account/AccountRepositoryInterface.php
View File

@ -66,10 +66,9 @@ interface AccountRepositoryInterface
* @param string $number
* @param array $types
*
* @deprecated
* @return Account
* @return Account|null
*/
public function findByAccountNumber(string $number, array $types): Account;
public function findByAccountNumber(string $number, array $types): ?Account;
/**
* @param string $iban

9
app/Repositories/Account/FindAccountsTrait.php
View File

@ -58,12 +58,9 @@ trait FindAccountsTrait
/**
* @param string $number
* @param array $types
*
*
* @deprecated
* @return Account
* @return Account|null
*/
public function findByAccountNumber(string $number, array $types): Account
public function findByAccountNumber(string $number, array $types): ?Account
{
$query = $this->user->accounts()
->leftJoin('account_meta', 'account_meta.account_id', '=', 'accounts.id')
@ -81,7 +78,7 @@ trait FindAccountsTrait
return $accounts->first();
}
return new Account;
return null;
}
/**

260
app/Support/Import/Placeholder/ImportTransaction.php
View File

@ -24,6 +24,11 @@ declare(strict_types=1);
namespace FireflyIII\Support\Import\Placeholder;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\Converter\Amount;
use FireflyIII\Import\Converter\AmountCredit;
use FireflyIII\Import\Converter\AmountDebit;
use FireflyIII\Import\Converter\ConverterInterface;
use Log;
/**
* Class ImportTransaction
@ -97,24 +102,6 @@ class ImportTransaction
/** @var array */
private $tags;
/**
* @return array
*/
public function getMeta(): array
{
return $this->meta;
}
/**
* @return string
*/
public function getNote(): string
{
return $this->note;
}
/**
* ImportTransaction constructor.
*/
@ -137,32 +124,6 @@ class ImportTransaction
}
/**
* @return string
*/
public function getDescription(): string
{
return $this->description;
}
/**
* @return int
*/
public function getBillId(): int
{
return $this->billId;
}
/**
* @return null|string
*/
public function getBillName(): ?string
{
return $this->billName;
}
/**
* @param ColumnValue $columnValue
*
@ -305,6 +266,135 @@ class ImportTransaction
}
}
/**
* Calculate the amount of this transaction.
*
* @return string
* @throws FireflyException
*/
public function calculateAmount(): string
{
Log::debug('Now in importTransaction->calculateAmount()');
$info = $this->selectAmountInput();
if (0 === \count($info)) {
throw new FireflyException('No amount information for this row.');
}
$class = $info['class'] ?? '';
if (0 === \strlen($class)) {
throw new FireflyException('No amount information (conversion class) for this row.');
}
Log::debug(sprintf('Converter class is %s', $info['class']));
/** @var ConverterInterface $amountConverter */
$amountConverter = app($info['class']);
$result = $amountConverter->convert($info['amount']);
Log::debug(sprintf('First attempt to convert gives "%s"', $result));
// modify
/**
* @var string $role
* @var string $modifier
*/
foreach ($this->modifiers as $role => $modifier) {
$class = sprintf('FireflyIII\Import\Converter\%s', config(sprintf('csv.import_roles.%s.converter', $role)));
/** @var ConverterInterface $converter */
$converter = app($class);
Log::debug(sprintf('Now launching converter %s', $class));
$conversion = $converter->convert($modifier);
if ($conversion === -1) {
$result = app('steam')->negative($result);
}
if ($conversion === 1) {
$result = app('steam')->positive($result);
}
Log::debug(sprintf('convertedAmount after conversion is %s', $result));
}
Log::debug(sprintf('After modifiers the result is: "%s"', $result));
return $result;
}
/**
* This array is being used to map the account the user is using.
*
* @return array
*/
public function getAccountData(): array
{
return [
'iban' => $this->accountIban,
'name' => $this->accountName,
'number' => $this->accountNumber,
'bic' => $this->accountBic,
];
}
/**
* @return int
*/
public function getAccountId(): int
{
return $this->accountId;
}
/**
* @return int
*/
public function getBillId(): int
{
return $this->billId;
}
/**
* @return null|string
*/
public function getBillName(): ?string
{
return $this->billName;
}
/**
* @return int
*/
public function getBudgetId(): int
{
return $this->budgetId;
}
/**
* @return string|null
*/
public function getBudgetName(): ?string
{
return $this->budgetName;
}
/**
* @return int
*/
public function getCategoryId(): int
{
return $this->categoryId;
}
/**
* @return string|null
*/
public function getCategoryName(): ?string
{
return $this->categoryName;
}
/**
* @return int
*/
public function getCurrencyId(): int
{
return $this->currencyId;
}
/**
* @return string
*/
@ -313,11 +403,64 @@ class ImportTransaction
return $this->date;
}
/**
* @return string
*/
public function getDescription(): string
{
return $this->description;
}
/**
* @return int
*/
public function getForeignCurrencyId(): int
{
return $this->foreignCurrencyId;
}
/**
* @return array
*/
public function getMeta(): array
{
return $this->meta;
}
/**
* @return string
*/
public function getNote(): string
{
return $this->note;
}
public function getOpposingAccountData(): array
{
return [
'iban' => $this->opposingIban,
'name' => $this->opposingName,
'number' => $this->opposingNumber,
'bic' => $this->opposingBic,
];
}
/**
* @return int
*/
public function getOpposingId(): int
{
return $this->opposingId;
}
/**
* @return array
*/
public function getTags(): array
{
return [];
// todo make sure this is an array
return $this->tags;
}
@ -333,4 +476,33 @@ class ImportTransaction
return $columnValue->getMappedValue() > 0 ? $columnValue->getMappedValue() : (int)$columnValue->getValue();
}
/**
* This methods decides which input value to use for the amount calculation.
*
* @return array
*/
private function selectAmountInput()
{
$info = [];
$converterClass = '';
if (null !== $this->amount) {
Log::debug('Amount value is not NULL, assume this is the correct value.');
$converterClass = Amount::class;
$info['amount'] = $this->amount;
}
if (null !== $this->amountDebit) {
Log::debug('Amount DEBIT value is not NULL, assume this is the correct value (overrules Amount).');
$converterClass = AmountDebit::class;
$info['amount'] = $this->amountDebit;
}
if (null !== $this->amountCredit) {
Log::debug('Amount CREDIT value is not NULL, assume this is the correct value (overrules Amount and AmountDebit).');
$converterClass = AmountCredit::class;
$info['amount'] = $this->amountCredit;
}
$info['class'] = $converterClass;
return $info;
}
}

346
app/Support/Import/Routine/File/CSVProcessor.php
View File

@ -26,6 +26,8 @@ namespace FireflyIII\Support\Import\Routine\File;
use Carbon\Carbon;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
use FireflyIII\Models\Account;
use FireflyIII\Models\AccountType;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
@ -50,6 +52,8 @@ use Log;
*/
class CSVProcessor implements FileProcessorInterface
{
/** @var AccountRepositoryInterface */
private $accountRepos;
/** @var AttachmentHelperInterface */
private $attachments;
/** @var array */
@ -69,7 +73,7 @@ class CSVProcessor implements FileProcessorInterface
*/
public function run(): array
{
Log::debug('Now in CSVProcessor() run');
// in order to actually map we also need to read the FULL file.
try {
@ -91,6 +95,7 @@ class CSVProcessor implements FileProcessorInterface
$array = $this->parseImportables($importables);
echo '<pre>';
print_r($array);
print_r($importables);
print_r($lines);
@ -103,11 +108,14 @@ class CSVProcessor implements FileProcessorInterface
*/
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->config = $job->configuration;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->attachments = app(AttachmentHelperInterface::class);
Log::debug('Now in setJob()');
$this->importJob = $job;
$this->config = $job->configuration;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->attachments = app(AttachmentHelperInterface::class);
$this->accountRepos = app(AccountRepositoryInterface::class);
$this->repository->setUser($job->user);
$this->accountRepos->setUser($job->user);
}
@ -121,6 +129,7 @@ class CSVProcessor implements FileProcessorInterface
*/
private function getLines(Reader $reader): array
{
Log::debug('now in getLines()');
$offset = isset($this->config['has-headers']) && $this->config['has-headers'] === true ? 1 : 0;
try {
$stmt = (new Statement)->offset($offset);
@ -143,6 +152,7 @@ class CSVProcessor implements FileProcessorInterface
*/
private function getReader(): Reader
{
Log::debug('Now in getReader()');
$content = '';
/** @var Collection $collection */
$collection = $this->importJob->attachments;
@ -195,28 +205,28 @@ class CSVProcessor implements FileProcessorInterface
case 'account-number':
$newRole = 'account-id';
break;
case 'foreign-currency-id':
case 'foreign-currency-code':
$newRole = 'foreign-currency-id';
break;
case 'bill-id':
case 'bill-name':
$newRole = 'bill-id';
break;
case 'budget-id':
case 'budget-name':
$newRole = 'budget-id';
break;
case 'currency-id':
case 'currency-name':
case 'currency-code':
case 'currency-symbol':
$newRole = 'currency-id';
break;
case 'budget-id':
case 'budget-name':
$newRole = 'budget-id';
break;
case 'category-id':
case 'category-name':
$newRole = 'category-id';
break;
case 'foreign-currency-id':
case 'foreign-currency-code':
$newRole = 'foreign-currency-id';
break;
case 'opposing-id':
case 'opposing-name':
case 'opposing-iban':
@ -232,6 +242,120 @@ class CSVProcessor implements FileProcessorInterface
return $newRole;
}
/**
* Based upon data in the importable, try to find or create the asset account account.
*
* @param $importable
*
* @return Account
*/
private function mapAssetAccount(?int $accountId, array $accountData): Account
{
Log::debug('Now in mapAssetAccount()');
if ((int)$accountId > 0) {
// find asset account with this ID:
$result = $this->accountRepos->findNull($accountId);
if (null !== $result) {
Log::debug(sprintf('Found account "%s" based on given ID %d. Return it!', $result->name, $accountId));
return $result;
}
}
// find by (respectively):
// IBAN, accountNumber, name,
$fields = ['iban' => 'findByIbanNull', 'number' => 'findByAccountNumber', 'name' => 'findByName'];
foreach ($fields as $field => $function) {
$value = $accountData[$field];
if (null === $value) {
continue;
}
$result = $this->accountRepos->$function($value, [AccountType::ASSET]);
Log::debug(sprintf('Going to run %s() with argument "%s" (asset account)', $function, $value));
if (null !== $result) {
Log::debug(sprintf('Found asset account "%s". Return it!', $result->name, $accountId));
return $result;
}
}
Log::debug('Found nothing. Will return default account.');
// still NULL? Return default account.
$default = null;
if (isset($this->config['import-account'])) {
$default = $this->accountRepos->findNull((int)$this->config['import-account']);
}
if (null === $default) {
Log::debug('Default account is NULL! Simply result first account in system.');
$default = $this->accountRepos->getAccountsByType([AccountType::ASSET])->first();
}
Log::debug(sprintf('Return default account "%s" (#%d). Return it!', $default->name, $default->id));
return $default;
}
/**
* @param int|null $accountId
* @param string $amount
* @param array $accountData
*
* @return Account
*/
private function mapOpposingAccount(?int $accountId, string $amount, array $accountData): Account
{
Log::debug('Now in mapOpposingAccount()');
if ((int)$accountId > 0) {
// find any account with this ID:
$result = $this->accountRepos->findNull($accountId);
if (null !== $result) {
Log::debug(sprintf('Found account "%s" (%s) based on given ID %d. Return it!', $result->name, $result->accountType->type, $accountId));
return $result;
}
}
// default assumption is we're looking for an expense account.
$expectedType = AccountType::EXPENSE;
Log::debug(sprintf('Going to search for accounts of type %s', $expectedType));
if (bccomp($amount, '0') === 1) {
// more than zero.
$expectedType = AccountType::REVENUE;
Log::debug(sprintf('Because amount is %s, will instead search for accounts of type %s', $amount, $expectedType));
}
// first search for $expectedType, then find asset:
$searchTypes = [$expectedType, AccountType::ASSET];
foreach ($searchTypes as $type) {
// find by (respectively):
// IBAN, accountNumber, name,
$fields = ['iban' => 'findByIbanNull', 'number' => 'findByAccountNumber', 'name' => 'findByName'];
foreach ($fields as $field => $function) {
$value = $accountData[$field];
if (null === $value) {
continue;
}
Log::debug(sprintf('Will search for account of type "%s" using %s() and argument %s.', $type, $function, $value));
$result = $this->accountRepos->$function($value, [$type]);
if (null !== $result) {
Log::debug(sprintf('Found result: Account #%d, named "%s"', $result->id, $result->name));
return $result;
}
}
}
// not found? Create it!
$creation = [
'name' => $accountData['name'],
'iban' => $accountData['iban'],
'accountNumber' => $accountData['number'],
'account_type_id' => null,
'accountType' => $expectedType,
'active' => true,
'BIC' => $accountData['bic'],
];
Log::debug('Will try to store a new account: ', $creation);
return $this->accountRepos->store($creation);
}
/**
* Each entry is an ImportTransaction that must be converted to an array compatible with the
* journal factory. To do so some stuff must still be resolved. See below.
@ -239,76 +363,119 @@ class CSVProcessor implements FileProcessorInterface
* @param array $importables
*
* @return array
* @throws FireflyException
*/
private function parseImportables(array $importables): array
{
Log::debug('Now in parseImportables()');
$array = [];
$total = count($importables);
/** @var ImportTransaction $importable */
foreach ($importables as $importable) {
// todo: verify bill mapping
// todo: verify currency mapping.
$entry = [
'type' => 'unknown', // todo
'date' => Carbon::createFromFormat($this->config['date-format'] ?? 'Ymd', $importable->getDate()),
'tags' => $importable->getTags(), // todo make sure its filled.
'user' => $this->importJob->user_id,
'notes' => $importable->getNote(),
// all custom fields:
'internal_reference' => $importable->getMeta()['internal-reference'] ?? null,
'sepa-cc' => $importable->getMeta()['sepa-cc'] ?? null,
'sepa-ct-op' => $importable->getMeta()['sepa-ct-op'] ?? null,
'sepa-ct-id' => $importable->getMeta()['sepa-ct-id'] ?? null,
'sepa-db' => $importable->getMeta()['sepa-db'] ?? null,
'sepa-country' => $importable->getMeta()['sepa-countru'] ?? null,
'sepa-ep' => $importable->getMeta()['sepa-ep'] ?? null,
'sepa-ci' => $importable->getMeta()['sepa-ci'] ?? null,
'interest_date' => $importable->getMeta()['date-interest'] ?? null,
'book_date' => $importable->getMeta()['date-book'] ?? null,
'process_date' => $importable->getMeta()['date-process'] ?? null,
'due_date' => $importable->getMeta()['date-due'] ?? null,
'payment_date' => $importable->getMeta()['date-payment'] ?? null,
'invoice_date' => $importable->getMeta()['date-invoice'] ?? null,
// todo external ID
// journal data:
'description' => $importable->getDescription(),
'piggy_bank_id' => null,
'piggy_bank_name' => null,
'bill_id' => $importable->getBillId() === 0 ? null : $importable->getBillId(), //
'bill_name' => $importable->getBillId() !== 0 ? null : $importable->getBillName(),
// transaction data:
'transactions' => [
[
'currency_id' => null, // todo find ma
'currency_code' => 'EUR',
'description' => null,
'amount' => random_int(500, 5000) / 100,
'budget_id' => null,
'budget_name' => null,
'category_id' => null,
'category_name' => null,
'source_id' => null,
'source_name' => 'Checking Account',
'destination_id' => null,
'destination_name' => 'Random expense account #' . random_int(1, 10000),
'foreign_currency_id' => null,
'foreign_currency_code' => null,
'foreign_amount' => null,
'reconciled' => false,
'identifier' => 0,
],
],
];
foreach ($importables as $index => $importable) {
Log::debug(sprintf('Now going to parse importable %d of %d', $index + 1, $total));
$array[] = $this->parseSingleImportable($importable);
}
return $array;
}
/**
* @param ImportTransaction $importable
*
* @return array
* @throws FireflyException
*/
private function parseSingleImportable(ImportTransaction $importable): array
{
$amount = $importable->calculateAmount();
/**
* first finalise the amount. cehck debit en credit.
* then get the accounts.
* ->account always assumes were looking for an asset account.
* cannot create anything, will return the default account when nothing comes up.
*
* neg + account = assume asset account?
* neg = assume withdrawal
* pos = assume
*/
$accountId = $this->verifyObjectId('account-id', $importable->getAccountId());
$billId = $this->verifyObjectId('bill-id', $importable->getForeignCurrencyId());
$budgetId = $this->verifyObjectId('budget-id', $importable->getBudgetId());
$currencyId = $this->verifyObjectId('currency-id', $importable->getForeignCurrencyId());
$categoryId = $this->verifyObjectId('category-id', $importable->getCategoryId());
$foreignCurrencyId = $this->verifyObjectId('foreign-currency-id', $importable->getForeignCurrencyId());
$opposingId = $this->verifyObjectId('opposing-id', $importable->getOpposingId());
// also needs amount to be final.
//$account = $this->mapAccount($accountId, $importable->getAccountData());
$asset = $this->mapAssetAccount($accountId, $importable->getAccountData());
$sourceId = $asset->id;
$opposing = $this->mapOpposingAccount($opposingId, $amount, $importable->getOpposingAccountData());
$destId = $opposing->id;
if (bccomp($amount, '0') === 1) {
// amount is positive? Then switch:
[$destId, $sourceId] = [$sourceId, $destId];
}
return [
'type' => 'unknown', // todo
'date' => Carbon::createFromFormat($this->config['date-format'] ?? 'Ymd', $importable->getDate()),
'tags' => $importable->getTags(), // todo make sure its filled.
'user' => $this->importJob->user_id,
'notes' => $importable->getNote(),
// all custom fields:
'internal_reference' => $importable->getMeta()['internal-reference'] ?? null,
'sepa-cc' => $importable->getMeta()['sepa-cc'] ?? null,
'sepa-ct-op' => $importable->getMeta()['sepa-ct-op'] ?? null,
'sepa-ct-id' => $importable->getMeta()['sepa-ct-id'] ?? null,
'sepa-db' => $importable->getMeta()['sepa-db'] ?? null,
'sepa-country' => $importable->getMeta()['sepa-countru'] ?? null,
'sepa-ep' => $importable->getMeta()['sepa-ep'] ?? null,
'sepa-ci' => $importable->getMeta()['sepa-ci'] ?? null,
'interest_date' => $importable->getMeta()['date-interest'] ?? null,
'book_date' => $importable->getMeta()['date-book'] ?? null,
'process_date' => $importable->getMeta()['date-process'] ?? null,
'due_date' => $importable->getMeta()['date-due'] ?? null,
'payment_date' => $importable->getMeta()['date-payment'] ?? null,
'invoice_date' => $importable->getMeta()['date-invoice'] ?? null,
// todo external ID
// journal data:
'description' => $importable->getDescription(),
'piggy_bank_id' => null,
'piggy_bank_name' => null,
'bill_id' => $billId,
'bill_name' => null === $budgetId ? $importable->getBillName() : null,
// transaction data:
'transactions' => [
[
'currency_id' => $currencyId, // todo what if null?
'currency_code' => null,
'description' => $importable->getDescription(),
'amount' => $amount,
'budget_id' => $budgetId,
'budget_name' => null === $budgetId ? $importable->getBudgetName() : null,
'category_id' => $categoryId,
'category_name' => null === $categoryId ? $importable->getCategoryName() : null,
'source_id' => $sourceId,
'source_name' => null,
'destination_id' => $destId,
'destination_name' => null,
'foreign_currency_id' => $foreignCurrencyId,
'foreign_currency_code' => null,
'foreign_amount' => null, // todo get me.
'reconciled' => false,
'identifier' => 0,
],
],
];
}
/**
* Process all lines in the CSV file. Each line is processed separately.
*
@ -319,6 +486,7 @@ class CSVProcessor implements FileProcessorInterface
*/
private function processLines(array $lines): array
{
Log::debug('Now in processLines()');
$processed = [];
$count = \count($lines);
foreach ($lines as $index => $line) {
@ -341,11 +509,14 @@ class CSVProcessor implements FileProcessorInterface
*/
private function processSingleLine(array $line): ImportTransaction
{
Log::debug('Now in processSingleLine()');
$transaction = new ImportTransaction;
// todo run all specifics on row.
foreach ($line as $column => $value) {
$value = trim($value);
$originalRole = $this->config['column-roles'][$column] ?? '_ignore';
Log::debug(sprintf('Now at column #%d (%s), value "%s"', $column, $originalRole, $value));
if (\strlen($value) > 0 && $originalRole !== '_ignore') {
// is a mapped value present?
@ -359,8 +530,9 @@ class CSVProcessor implements FileProcessorInterface
$columnValue->setMappedValue($mapped);
$columnValue->setOriginalRole($originalRole);
$transaction->addColumnValue($columnValue);
Log::debug(sprintf('Now at column #%d (%s), value "%s"', $column, $role, $value));
}
if ('' === $value) {
Log::debug('Column skipped because value is empty.');
}
}
@ -377,6 +549,7 @@ class CSVProcessor implements FileProcessorInterface
*/
private function validateMappedValues()
{
Log::debug('Now in validateMappedValues()');
foreach ($this->mappedValues as $role => $values) {
$values = array_unique($values);
if (count($values) > 0) {
@ -385,10 +558,7 @@ class CSVProcessor implements FileProcessorInterface
throw new FireflyException(sprintf('Cannot validate mapped values for role "%s"', $role));
case 'opposing-id':
case 'account-id':
/** @var AccountRepositoryInterface $repository */
$repository = app(AccountRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$set = $repository->getAccountsById($values);
$set = $this->accountRepos->getAccountsById($values);
$valid = $set->pluck('id')->toArray();
$this->mappedValues[$role] = $valid;
break;
@ -429,4 +599,22 @@ class CSVProcessor implements FileProcessorInterface
}
}
}
/**
* A small function that verifies if this particular key (ID) is present in the list
* of valid keys.
*
* @param string $key
* @param int $objectId
*
* @return int|null
*/
private function verifyObjectId(string $key, int $objectId): ?int
{
if (isset($this->mappedValues[$key]) && in_array($objectId, $this->mappedValues[$key])) {
return $objectId;
}
return null;
}
}