IntenseFinance/firefly-iii
3
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2025-02-25 18:45:27 -06:00
Code Issues Packages Projects Releases Wiki Activity

Improve CSV import routine.

Browse Source
This commit is contained in:
James Cole 2018-05-07 19:21:12 +02:00
parent 690c9203c8
commit 626f7357bb
No known key found for this signature in database
GPG Key ID: C16961E655E74B5E
4 changed files with 435 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Import/Routine/FileRoutine.php
View File

@ -26,7 +26,6 @@ use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Routine\File\FileProcessorInterface;
use FireflyIII\Support\Import\Routine\File\FileRoutineInterface;
use Log;
/**

113
app/Support/Import/Placeholder/ColumnValue.php Normal file
View File

@ -0,0 +1,113 @@
<?php
/**
* Column.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Placeholder;
/**
* Class Column
*/
class ColumnValue
{
/** @var int */
private $mappedValue;
/** @var string */
private $originalRole;
/** @var string */
private $role;
/** @var string */
private $value;
/**
* ColumnValue constructor.
*/
public function __construct()
{
$this->mappedValue = 0;
}
/**
* @return int
*/
public function getMappedValue(): int
{
return $this->mappedValue;
}
/**
* @param int $mappedValue
*/
public function setMappedValue(int $mappedValue): void
{
$this->mappedValue = $mappedValue;
}
/**
* @return string
*/
public function getOriginalRole(): string
{
return $this->originalRole;
}
/**
* @param string $originalRole
*/
public function setOriginalRole(string $originalRole): void
{
$this->originalRole = $originalRole;
}
/**
* @return string
*/
public function getRole(): string
{
return $this->role;
}
/**
* @param string $role
*/
public function setRole(string $role): void
{
$this->role = $role;
}
/**
* @return string
*/
public function getValue(): string
{
return $this->value;
}
/**
* @param string $value
*/
public function setValue(string $value): void
{
$this->value = $value;
}
}

220
app/Support/Import/Placeholder/ImportTransaction.php Normal file
View File

@ -0,0 +1,220 @@
<?php
/**
* Transaction.php
* Copyright (c) 2018 thegrumpydictator@gmail.com
*
* This file is part of Firefly III.
*
* Firefly III is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Firefly III is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Firefly III. If not, see <http://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace FireflyIII\Support\Import\Placeholder;
use FireflyIII\Exceptions\FireflyException;
/**
* Class ImportTransaction
*/
class ImportTransaction
{
/** @var string */
private $accountBic;
/** @var string */
private $accountIban;
/** @var int */
private $accountId;
/** @var string */
private $accountName;
/** @var string */
private $amount;
/** @var string */
private $amountCredit;
/** @var string */
private $amountDebit;
/** @var int */
private $billId;
/** @var int */
private $budgetId;
/** @var int */
private $categoryId;
/** @var int */
private $currencyId;
/** @var string */
private $date;
/** @var string */
private $description;
/** @var string */
private $externalId;
/** @var string */
private $foreignAmount;
/** @var int */
private $foreignCurrencyId;
/** @var array */
private $meta;
/** @var array */
private $modifiers;
/** @var string */
private $note;
/** @var string */
private $opposingBic;
/** @var string */
private $opposingIban;
/** @var int */
private $opposingId;
/** @var string */
private $opposingName;
/** @var array */
private $tags;
/**
* ImportTransaction constructor.
*/
public function __construct()
{
$this->tags = [];
$this->modifiers = [];
$this->meta = [];
$this->description = '';
$this->note = '';
}
/**
* @param ColumnValue $columnValue
*
* @throws FireflyException
*/
public function addColumnValue(ColumnValue $columnValue): void
{
switch ($columnValue->getRole()) {
default:
throw new FireflyException(
sprintf('ImportTransaction cannot handle role "%s" with value "%s"', $columnValue->getRole(), $columnValue->getValue())
);
case 'account-id':
// could be the result of a mapping?
$this->accountId = $this->getMappedValue($columnValue);
break;
case 'account-name':
$this->accountName = $columnValue->getValue();
break;
case 'sepa-ct-id';
case 'sepa-ct-op';
case 'sepa-db';
case 'sepa-cc':
case 'sepa-country';
case 'sepa-ep';
case 'sepa-ci';
case 'internal-reference':
case 'date-interest':
case 'date-invoice':
case 'date-book':
case 'date-payment':
case 'date-process':
case 'date-due':
$this->meta[$columnValue->getRole()] = $columnValue->getValue();
break;
case'amount_debit':
$this->amountDebit = $columnValue->getValue();
break;
case'amount_credit':
$this->amountCredit = $columnValue->getValue();
break;
case 'amount':
$this->amount = $columnValue->getValue();
break;
case 'amount_foreign':
$this->foreignAmount = $columnValue->getValue();
break;
case 'foreign-currency-id':
$this->foreignCurrencyId = $this->getMappedValue($columnValue);
break;
case 'bill-id':
$this->billId = $this->getMappedValue($columnValue);
break;
case 'budget-id':
$this->budgetId = $this->getMappedValue($columnValue);
break;
case 'category-id':
$this->categoryId = $this->getMappedValue($columnValue);
break;
case 'currency-id':
$this->currencyId = $this->getMappedValue($columnValue);
break;
case 'date-transaction':
$this->date = $columnValue->getValue();
break;
case 'description':
$this->description .= $columnValue->getValue();
break;
case 'note':
$this->note .= $columnValue->getValue();
break;
case 'external-id':
$this->externalId = $columnValue->getValue();
break;
case 'rabo-debit-credit':
case 'ing-debit-credit':
$this->modifiers[$columnValue->getRole()] = $columnValue->getValue();
break;
case 'opposing-id':
$this->opposingId = $this->getMappedValue($columnValue);
break;
case 'opposing-name':
$this->opposingName = $columnValue->getValue();
break;
case 'opposing-bic':
$this->opposingBic = $columnValue->getValue();
break;
case 'tags-comma':
// todo split using pre-processor.
$this->tags = $columnValue->getValue();
break;
case 'tags-space':
// todo split using pre-processor.
$this->tags = $columnValue->getValue();
break;
case 'account-iban':
$this->accountIban = $columnValue->getValue();
break;
case 'opposing-iban':
$this->opposingIban = $columnValue->getValue();
break;
case '_ignore':
case 'bill-name':
case 'currency-name':
case 'currency-code':
case 'foreign-currency-code':
case 'currency-symbol':
case 'budget-name':
case 'category-name':
case 'account-number':
case 'opposing-number':
}
}
/**
* Returns the mapped value if it exists in the ColumnValue object.
*
* @param ColumnValue $columnValue
*
* @return int
*/
private function getMappedValue(ColumnValue $columnValue): int
{
return $columnValue->getMappedValue() > 0 ? $columnValue->getMappedValue() : (int)$columnValue->getValue();
}
}

137
app/Support/Import/Routine/File/CSVProcessor.php
View File

@ -28,6 +28,8 @@ use FireflyIII\Helpers\Attachments\AttachmentHelperInterface;
use FireflyIII\Models\Attachment;
use FireflyIII\Models\ImportJob;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Support\Import\Placeholder\ColumnValue;
use FireflyIII\Support\Import\Placeholder\ImportTransaction;
use Illuminate\Support\Collection;
use League\Csv\Exception;
use League\Csv\Reader;
@ -44,8 +46,12 @@ class CSVProcessor implements FileProcessorInterface
{
/** @var AttachmentHelperInterface */
private $attachments;
/** @var array */
private $config;
/** @var ImportJob */
private $importJob;
/** @var array */
private $mappedValues;
/** @var ImportJobRepositoryInterface */
private $repository;
@ -57,7 +63,7 @@ class CSVProcessor implements FileProcessorInterface
*/
public function run(): array
{
$config = $this->importJob->configuration;
// in order to actually map we also need to read the FULL file.
try {
@ -66,14 +72,11 @@ class CSVProcessor implements FileProcessorInterface
Log::error($e->getMessage());
throw new FireflyException('Cannot get reader: ' . $e->getMessage());
}
// get mapping from config
$roles = $config['column-roles'];
// get all lines from file:
$lines = $this->getLines($reader, $config);
$lines = $this->getLines($reader);
// make import objects, according to their role:
$importables = $this->processLines($lines, $roles);
$importables = $this->processLines($lines);
echo '<pre>';
print_r($importables);
@ -89,6 +92,7 @@ class CSVProcessor implements FileProcessorInterface
public function setJob(ImportJob $job): void
{
$this->importJob = $job;
$this->config = $job->configuration;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->attachments = app(AttachmentHelperInterface::class);
$this->repository->setUser($job->user);
@ -99,14 +103,13 @@ class CSVProcessor implements FileProcessorInterface
* Returns all lines from the CSV file.
*
* @param Reader $reader
* @param array $config
*
* @return array
* @throws FireflyException
*/
private function getLines(Reader $reader, array $config): array
private function getLines(Reader $reader): array
{
$offset = isset($config['has-headers']) && $config['has-headers'] === true ? 1 : 0;
$offset = isset($this->config['has-headers']) && $this->config['has-headers'] === true ? 1 : 0;
try {
$stmt = (new Statement)->offset($offset);
} catch (Exception $e) {
@ -146,45 +149,93 @@ class CSVProcessor implements FileProcessorInterface
}
/**
* Process a single column. Return is an array with:
* [0 => key, 1 => value]
* where the first item is the key under which the value
* must be stored, and the second is the value.
* If the value in the column is mapped to a certain ID,
* the column where this ID must be placed will change.
*
* @param int $column
* @param string $value
* @param string $role
* For example, if you map role "budget-name" with value "groceries" to 1,
* then that should become the budget-id. Not the name.
*
* @return array
* @param int $column
* @param int $mapped
*
* @return string
* @throws FireflyException
*/
private function processColumn(int $column, string $value, string $role): array
private function getRoleForColumn(int $column, int $mapped): string
{
$roles = $this->config['column-roles'];
$role = $roles[$column] ?? '_ignore';
if ($mapped === 0) {
Log::debug(sprintf('Column #%d with role "%s" is not mapped.', $column, $role));
return $role;
}
if (!(isset($this->config['column-do-mapping'][$column]) && $this->config['column-do-mapping'][$column] === true)) {
return $role;
}
switch ($role) {
default:
throw new FireflyException(sprintf('Cannot handle role "%s" with value "%s"', $role, $value));
// feed each line into a new class which will process
// the line.
throw new FireflyException(sprintf('Cannot indicate new role for mapped role "%s"', $role));
case 'account-id':
case 'account-name':
case 'account-iban':
case 'account-number':
$newRole = 'account-id';
break;
case 'foreign-currency-id':
case 'foreign-currency-code':
$newRole = 'foreign-currency-id';
break;
case 'bill-id':
case 'bill-name':
$newRole = 'bill-id';
break;
case 'currency-id':
case 'currency-name':
case 'currency-code':
case 'currency-symbol':
$newRole = 'currency-id';
break;
case 'budget-id':
case 'budget-name':
$newRole = 'budget-id';
break;
case 'category-id':
case 'category-name':
$newRole = 'category-id';
break;
case 'opposing-id':
case 'opposing-name':
case 'opposing-iban':
case 'opposing-number':
$newRole = 'opposing-id';
break;
}
Log::debug(sprintf('Role was "%s", but because of mapping, role becomes "%s"', $role, $newRole));
// also store the $mapped values in a "mappedValues" array.
$this->mappedValues[$newRole] = $mapped;
return $newRole;
}
/**
* Process all lines in the CSV file. Each line is processed separately.
*
* @param array $lines
* @param array $roles
*
* @return array
* @throws FireflyException
*/
private function processLines(array $lines, array $roles): array
private function processLines(array $lines): array
{
$processed = [];
foreach ($lines as $line) {
$processed[] = $this->processSingleLine($line, $roles);
$count = \count($lines);
foreach ($lines as $index => $line) {
Log::debug(sprintf('Now at line #%d of #%d', $index, $count));
$processed[] = $this->processSingleLine($line);
}
return $processed;
@ -197,18 +248,34 @@ class CSVProcessor implements FileProcessorInterface
* @param array $line
* @param array $roles
*
* @return array
* @return ImportTransaction
* @throws FireflyException
*/
private function processSingleLine(array $line, array $roles): array
private function processSingleLine(array $line): ImportTransaction
{
$transaction = new ImportTransaction;
// todo run all specifics on row.
$transaction = [];
foreach ($line as $column => $value) {
$value = trim($value);
$role = $roles[$column] ?? '_ignore';
[$key, $result] = $this->processColumn($column, $value, $role);
// if relevant, find mapped value:
$value = trim($value);
$originalRole = $this->config['column-roles'][$column] ?? '_ignore';
if (\strlen($value) > 0 && $originalRole !== '_ignore') {
// is a mapped value present?
$mapped = $this->config['column-mapping-config'][$column][$value] ?? 0;
// the role might change.
$role = $this->getRoleForColumn($column, $mapped);
$columnValue = new ColumnValue;
$columnValue->setValue($value);
$columnValue->setRole($role);
$columnValue->setMappedValue($mapped);
$columnValue->setOriginalRole($originalRole);
$transaction->addColumnValue($columnValue);
// create object that parses this column value.
Log::debug(sprintf('Now at column #%d (%s), value "%s"', $column, $role, $value));
}
}
return $transaction;