mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2025-02-25 18:45:27 -06:00
Further improve import routine
This commit is contained in:
James Cole
parent
a905cce2c9
commit
cddaccb7f7
@ -19,6 +19,7 @@ use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Models\TransactionJournal;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\MessageBag;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
@ -79,9 +80,17 @@ class Import extends Command
|
||||
$routine = new ImportRoutine($job);
|
||||
$routine->run();
|
||||
|
||||
/** @var MessageBag $error */
|
||||
foreach($routine->errors as $index => $error) {
|
||||
if($error->count() > 0) {
|
||||
$message = join(', ',$error->all());
|
||||
$this->error(sprintf('Error importing line #%d: %s', $index, $message));
|
||||
}
|
||||
}
|
||||
|
||||
// display result to user:
|
||||
//$this->presentResults($result);
|
||||
$this->line('The import has completed.');
|
||||
$this->line(sprintf('The import has finished. %d transactions have been imported out of %d records.', $routine->journals->count(), $routine->lines));
|
||||
|
||||
// get any errors from the importer:
|
||||
//$this->presentErrors($job);
|
||||
|
||||
@ -64,9 +64,8 @@ class CsvProcessor implements FileProcessorInterface
|
||||
*/
|
||||
public function run(): bool
|
||||
{
|
||||
// TODO update the job and say we started:
|
||||
//$this->job->status = 'running';
|
||||
//$this->job->save();
|
||||
$this->job->status = 'running';
|
||||
$this->job->save();
|
||||
Log::debug('Now in CsvProcessor run(). Job is now running...');
|
||||
|
||||
$entries = $this->getImportArray();
|
||||
|
||||
@ -12,13 +12,52 @@ declare(strict_types=1);
|
||||
namespace FireflyIII\Import\Object;
|
||||
|
||||
|
||||
use FireflyIII\Models\Bill;
|
||||
use FireflyIII\Repositories\Bill\BillRepositoryInterface;
|
||||
use FireflyIII\User;
|
||||
use Illuminate\Support\Collection;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class ImportBill
|
||||
*
|
||||
* @package FireflyIII\Import\Object
|
||||
*/
|
||||
class ImportBill
|
||||
{
|
||||
|
||||
/** @var Bill */
|
||||
private $bill;
|
||||
/** @var array */
|
||||
private $id = [];
|
||||
/** @var array */
|
||||
private $name = [];
|
||||
/** @var BillRepositoryInterface */
|
||||
private $repository;
|
||||
/** @var User */
|
||||
private $user;
|
||||
|
||||
/**
|
||||
* ImportBill constructor.
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->bill = new Bill;
|
||||
$this->repository = app(BillRepositoryInterface::class);
|
||||
Log::debug('Created ImportBill.');
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Bill
|
||||
*/
|
||||
public function getBill(): Bill
|
||||
{
|
||||
if (is_null($this->bill->id)) {
|
||||
$this->store();
|
||||
}
|
||||
|
||||
return $this->bill;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $id
|
||||
@ -36,5 +75,154 @@ class ImportBill
|
||||
$this->name = $name;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param User $user
|
||||
*/
|
||||
public function setUser(User $user)
|
||||
{
|
||||
$this->user = $user;
|
||||
$this->repository->setUser($user);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Bill
|
||||
*/
|
||||
private function findExistingObject(): Bill
|
||||
{
|
||||
Log::debug('In findExistingObject() for Bill');
|
||||
// 1: find by ID, or name
|
||||
|
||||
if (count($this->id) === 3) {
|
||||
Log::debug(sprintf('Finding bill with ID #%d', $this->id['value']));
|
||||
/** @var Bill $bill */
|
||||
$bill = $this->repository->find(intval($this->id['value']));
|
||||
if (!is_null($bill->id)) {
|
||||
Log::debug(sprintf('Found unmapped bill by ID (#%d): %s', $bill->id, $bill->name));
|
||||
|
||||
return $bill;
|
||||
}
|
||||
Log::debug('Found nothing.');
|
||||
}
|
||||
// 2: find by name
|
||||
if (count($this->name) === 3) {
|
||||
/** @var Collection $bills */
|
||||
$bills = $this->repository->getBills();
|
||||
$name = $this->name['value'];
|
||||
Log::debug(sprintf('Finding bill with name %s', $name));
|
||||
$filtered = $bills->filter(
|
||||
function (Bill $bill) use ($name) {
|
||||
if ($bill->name === $name) {
|
||||
Log::debug(sprintf('Found unmapped bill by name (#%d): %s', $bill->id, $bill->name));
|
||||
|
||||
return $bill;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
);
|
||||
|
||||
if ($filtered->count() === 1) {
|
||||
return $filtered->first();
|
||||
}
|
||||
Log::debug('Found nothing.');
|
||||
}
|
||||
|
||||
// 4: do not search by account number.
|
||||
Log::debug('Found NO existing bills.');
|
||||
|
||||
return new Bill;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Bill
|
||||
*/
|
||||
private function findMappedObject(): Bill
|
||||
{
|
||||
Log::debug('In findMappedObject() for Bill');
|
||||
$fields = ['id', 'name'];
|
||||
foreach ($fields as $field) {
|
||||
$array = $this->$field;
|
||||
Log::debug(sprintf('Find mapped bill based on field "%s" with value', $field), $array);
|
||||
// check if a pre-mapped object exists.
|
||||
$mapped = $this->getMappedObject($array);
|
||||
if (!is_null($mapped->id)) {
|
||||
Log::debug(sprintf('Found bill #%d!', $mapped->id));
|
||||
|
||||
return $mapped;
|
||||
}
|
||||
|
||||
}
|
||||
Log::debug('Found no bill on mapped data or no map present.');
|
||||
|
||||
return new Bill;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $array
|
||||
*
|
||||
* @return Bill
|
||||
*/
|
||||
private function getMappedObject(array $array): Bill
|
||||
{
|
||||
Log::debug('In getMappedObject() for Bill');
|
||||
if (count($array) === 0) {
|
||||
Log::debug('Array is empty, nothing will come of this.');
|
||||
|
||||
return new Bill;
|
||||
}
|
||||
|
||||
if (array_key_exists('mapped', $array) && is_null($array['mapped'])) {
|
||||
Log::debug(sprintf('No map present for value "%s". Return NULL.', $array['value']));
|
||||
|
||||
return new Bill;
|
||||
}
|
||||
|
||||
Log::debug('Finding a mapped bill based on', $array);
|
||||
|
||||
$search = intval($array['mapped']);
|
||||
$account = $this->repository->find($search);
|
||||
|
||||
Log::debug(sprintf('Found bill! #%d ("%s"). Return it', $account->id, $account->name));
|
||||
|
||||
return $account;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
private function store(): bool
|
||||
{
|
||||
// 1: find mapped object:
|
||||
$mapped = $this->findMappedObject();
|
||||
if (!is_null($mapped->id)) {
|
||||
$this->bill = $mapped;
|
||||
|
||||
return true;
|
||||
}
|
||||
// 2: find existing by given values:
|
||||
$found = $this->findExistingObject();
|
||||
if (!is_null($found->id)) {
|
||||
$this->bill = $found;
|
||||
|
||||
return true;
|
||||
}
|
||||
$name = $this->name['value'] ?? '';
|
||||
|
||||
if (strlen($name) === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
Log::debug('Found no bill so must create one ourselves.');
|
||||
|
||||
$data = [
|
||||
'name' => $name,
|
||||
];
|
||||
|
||||
$this->bill = $this->repository->store($data);
|
||||
Log::debug(sprintf('Successfully stored new bill #%d: %s', $this->bill->id, $this->bill->name));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
@ -33,7 +33,7 @@ class ImportJournal
|
||||
/** @var ImportBudget */
|
||||
public $budget;
|
||||
/** @var string */
|
||||
public $description;
|
||||
public $description = '';
|
||||
/** @var Collection */
|
||||
public $errors;
|
||||
/** @var string */
|
||||
@ -43,7 +43,7 @@ class ImportJournal
|
||||
/** @var string */
|
||||
private $amount = '0';
|
||||
/** @var ImportBill */
|
||||
private $bill;
|
||||
public $bill;
|
||||
/** @var ImportCategory */
|
||||
public $category;
|
||||
/** @var ImportCurrency */
|
||||
@ -54,11 +54,16 @@ class ImportJournal
|
||||
private $externalId = '';
|
||||
/** @var array */
|
||||
private $modifiers = [];
|
||||
/** @var array */
|
||||
private $tags = [];
|
||||
/** @var string */
|
||||
public $notes = '';
|
||||
/** @var string */
|
||||
private $transactionType = '';
|
||||
/** @var User */
|
||||
private $user;
|
||||
/** @var array */
|
||||
public $metaDates = [];
|
||||
|
||||
/**
|
||||
* ImportEntry constructor.
|
||||
@ -151,6 +156,7 @@ class ImportJournal
|
||||
$this->opposing->setUser($user);
|
||||
$this->budget->setUser($user);
|
||||
$this->category->setUser($user);
|
||||
$this->bill->setUser($user);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -214,6 +220,12 @@ class ImportJournal
|
||||
case 'description':
|
||||
$this->description = $array['value'];
|
||||
break;
|
||||
case 'sepa-ct-op':
|
||||
case 'sepa-ct-id':
|
||||
case 'sepa-db':
|
||||
$this->notes .= ' '.$array['value'];
|
||||
$this->notes = trim($this->notes);
|
||||
break;
|
||||
case 'external-id':
|
||||
$this->externalId = $array['value'];
|
||||
break;
|
||||
@ -239,6 +251,16 @@ class ImportJournal
|
||||
case 'tags-space':
|
||||
$this->tags[] = $array;
|
||||
break;
|
||||
// 'interest_date', 'book_date', 'process_date', 'due_date', 'payment_date', 'invoice_date',
|
||||
case 'date-interest':
|
||||
$this->metaDates['interest_date'] = $array['value'];
|
||||
break;
|
||||
case 'date-book':
|
||||
$this->metaDates['book_date'] = $array['value'];
|
||||
break;
|
||||
case 'date-process':
|
||||
$this->metaDates['process_date'] = $array['value'];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -21,6 +21,12 @@ use Log;
|
||||
class ImportRoutine
|
||||
{
|
||||
|
||||
/** @var Collection */
|
||||
public $journals;
|
||||
/** @var Collection */
|
||||
public $errors;
|
||||
/** @var int */
|
||||
public $lines = 0;
|
||||
/** @var ImportJob */
|
||||
private $job;
|
||||
|
||||
@ -31,7 +37,9 @@ class ImportRoutine
|
||||
*/
|
||||
public function __construct(ImportJob $job)
|
||||
{
|
||||
$this->job = $job;
|
||||
$this->job = $job;
|
||||
$this->journals = new Collection;
|
||||
$this->errors = new Collection;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -58,6 +66,9 @@ class ImportRoutine
|
||||
$processor->run();
|
||||
$objects = $processor->getObjects();
|
||||
}
|
||||
$this->lines = $objects->count();
|
||||
// once done, use storage thing to actually store them:
|
||||
Log::debug(sprintf('Returned %d valid objects from file processor', $this->lines));
|
||||
|
||||
$storage = new ImportStorage;
|
||||
$storage->setJob($this->job);
|
||||
@ -65,8 +76,12 @@ class ImportRoutine
|
||||
$storage->setObjects($objects);
|
||||
$storage->store();
|
||||
|
||||
// once done, use storage thing to actually store them:
|
||||
Log::debug(sprintf('Returned %d valid objects from file processor', $objects->count()));
|
||||
// update job:
|
||||
$this->job->status = 'finished';
|
||||
$this->job->save();
|
||||
|
||||
$this->journals = $storage->journals;
|
||||
$this->errors = $storage->errors;
|
||||
|
||||
Log::debug(sprintf('Done with import job %s', $this->job->key));
|
||||
|
||||
|
||||
@ -12,15 +12,17 @@ declare(strict_types=1);
|
||||
namespace FireflyIII\Import\Storage;
|
||||
|
||||
use Amount;
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use Carbon\Carbon;
|
||||
use FireflyIII\Import\Object\ImportJournal;
|
||||
use FireflyIII\Models\Account;
|
||||
use FireflyIII\Models\AccountType;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Models\Transaction;
|
||||
use FireflyIII\Models\TransactionCurrency;
|
||||
use FireflyIII\Models\TransactionJournal;
|
||||
use FireflyIII\Models\TransactionType;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\MessageBag;
|
||||
use Log;
|
||||
use Steam;
|
||||
|
||||
@ -32,8 +34,12 @@ use Steam;
|
||||
*/
|
||||
class ImportStorage
|
||||
{
|
||||
/** @var Collection */
|
||||
public $errors;
|
||||
public $journals;
|
||||
/** @var string */
|
||||
private $dateFormat = 'Ymd';
|
||||
/** @var TransactionCurrency */
|
||||
private $defaultCurrency;
|
||||
/** @var ImportJob */
|
||||
private $job;
|
||||
@ -45,7 +51,9 @@ class ImportStorage
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->objects = new Collection;
|
||||
$this->objects = new Collection;
|
||||
$this->journals = new Collection;
|
||||
$this->errors = new Collection;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -87,6 +95,8 @@ class ImportStorage
|
||||
foreach ($this->objects as $index => $object) {
|
||||
Log::debug(sprintf('Going to store object #%d with description "%s"', $index, $object->description));
|
||||
|
||||
$errors = new MessageBag;
|
||||
|
||||
// create the asset account
|
||||
$asset = $object->asset->getAccount();
|
||||
$opposing = new Account;
|
||||
@ -131,7 +141,10 @@ class ImportStorage
|
||||
$journal->encrypted = 0;
|
||||
$journal->completed = 0;
|
||||
if (!$journal->save()) {
|
||||
throw new FireflyException($journal->getErrors()->first());
|
||||
$errorText = join(', ', $journal->getErrors()->all());
|
||||
$errors->add('no-key', sprintf('Error storing journal: %s', $errorText));
|
||||
Log::error(sprintf('Could not store line #%d: %s', $index, $errorText));
|
||||
continue;
|
||||
}
|
||||
$journal->setMeta('importHash', $object->hash);
|
||||
Log::debug(sprintf('Created journal with ID #%d', $journal->id));
|
||||
@ -143,6 +156,10 @@ class ImportStorage
|
||||
$one->transaction_currency_id = $currency->id;
|
||||
$one->amount = $amount;
|
||||
$one->save();
|
||||
if (is_null($one->id)) {
|
||||
$errorText = join(', ', $one->getErrors()->all());
|
||||
$errors->add('no-key', sprintf('Error storing transaction one for journal %d: %s', $journal->id, $errorText));
|
||||
}
|
||||
Log::debug(sprintf('Created transaction with ID #%d and account #%d', $one->id, $asset->id));
|
||||
|
||||
$two = new Transaction;
|
||||
@ -151,6 +168,10 @@ class ImportStorage
|
||||
$two->transaction_currency_id = $currency->id;
|
||||
$two->amount = Steam::opposite($amount);
|
||||
$two->save();
|
||||
if (is_null($two->id)) {
|
||||
$errorText = join(', ', $two->getErrors()->all());
|
||||
$errors->add('no-key', sprintf('Error storing transaction one for journal %d: %s', $journal->id, $errorText));
|
||||
}
|
||||
Log::debug(sprintf('Created transaction with ID #%d and account #%d', $two->id, $opposing->id));
|
||||
|
||||
// category
|
||||
@ -167,13 +188,33 @@ class ImportStorage
|
||||
$journal->budgets()->save($budget);
|
||||
}
|
||||
// bill
|
||||
$bill = $object->bill->getBill();
|
||||
if (!is_null($bill->id)) {
|
||||
Log::debug(sprintf('Linked bill #%d to journal #%d', $bill->id, $journal->id));
|
||||
$journal->bill()->associate($bill);
|
||||
$journal->save();
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
// all other date fields as meta thing:
|
||||
foreach ($object->metaDates as $name => $value) {
|
||||
try {
|
||||
$date = new Carbon($value);
|
||||
$journal->setMeta($name, $date);
|
||||
} catch (\Exception $e) {
|
||||
// don't care, ignore:
|
||||
Log::warning(sprintf('Could not parse "%s" into a valid Date object for field %s', $value, $name));
|
||||
}
|
||||
}
|
||||
|
||||
// sepa thing as note:
|
||||
if (strlen($object->notes) > 0) {
|
||||
$journal->setMeta('notes', $object->notes);
|
||||
}
|
||||
$this->journals->push($journal);
|
||||
$this->errors->push($errors);
|
||||
}
|
||||
|
||||
die('Cannot actually store yet.');
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -46,6 +46,7 @@ class ImportJob extends Model
|
||||
'initialized',
|
||||
'configured',
|
||||
'running',
|
||||
'finished',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
Loading…
Reference in New Issue
Block a user