diff --git a/app/Console/Commands/Import.php b/app/Console/Commands/Import.php index 6b54233707..e5fbaee518 100644 --- a/app/Console/Commands/Import.php +++ b/app/Console/Commands/Import.php @@ -19,6 +19,7 @@ use FireflyIII\Models\ImportJob; use FireflyIII\Models\TransactionJournal; use Illuminate\Console\Command; use Illuminate\Support\Collection; +use Illuminate\Support\MessageBag; use Log; /** @@ -79,9 +80,17 @@ class Import extends Command $routine = new ImportRoutine($job); $routine->run(); + /** @var MessageBag $error */ + foreach($routine->errors as $index => $error) { + if($error->count() > 0) { + $message = join(', ',$error->all()); + $this->error(sprintf('Error importing line #%d: %s', $index, $message)); + } + } + // display result to user: //$this->presentResults($result); - $this->line('The import has completed.'); + $this->line(sprintf('The import has finished. %d transactions have been imported out of %d records.', $routine->journals->count(), $routine->lines)); // get any errors from the importer: //$this->presentErrors($job); diff --git a/app/Import/FileProcessor/CsvProcessor.php b/app/Import/FileProcessor/CsvProcessor.php index 9120ff2e34..92329c980e 100644 --- a/app/Import/FileProcessor/CsvProcessor.php +++ b/app/Import/FileProcessor/CsvProcessor.php @@ -64,9 +64,8 @@ class CsvProcessor implements FileProcessorInterface */ public function run(): bool { - // TODO update the job and say we started: - //$this->job->status = 'running'; - //$this->job->save(); + $this->job->status = 'running'; + $this->job->save(); Log::debug('Now in CsvProcessor run(). Job is now running...'); $entries = $this->getImportArray(); diff --git a/app/Import/Object/ImportBill.php b/app/Import/Object/ImportBill.php index 40c511e1d4..c3d5f2f93d 100644 --- a/app/Import/Object/ImportBill.php +++ b/app/Import/Object/ImportBill.php @@ -12,13 +12,52 @@ declare(strict_types=1); namespace FireflyIII\Import\Object; +use FireflyIII\Models\Bill; +use FireflyIII\Repositories\Bill\BillRepositoryInterface; +use FireflyIII\User; +use Illuminate\Support\Collection; +use Log; + +/** + * Class ImportBill + * + * @package FireflyIII\Import\Object + */ class ImportBill { + /** @var Bill */ + private $bill; /** @var array */ private $id = []; /** @var array */ private $name = []; + /** @var BillRepositoryInterface */ + private $repository; + /** @var User */ + private $user; + + /** + * ImportBill constructor. + */ + public function __construct() + { + $this->bill = new Bill; + $this->repository = app(BillRepositoryInterface::class); + Log::debug('Created ImportBill.'); + } + + /** + * @return Bill + */ + public function getBill(): Bill + { + if (is_null($this->bill->id)) { + $this->store(); + } + + return $this->bill; + } /** * @param array $id @@ -36,5 +75,154 @@ class ImportBill $this->name = $name; } + /** + * @param User $user + */ + public function setUser(User $user) + { + $this->user = $user; + $this->repository->setUser($user); + } + + /** + * @return Bill + */ + private function findExistingObject(): Bill + { + Log::debug('In findExistingObject() for Bill'); + // 1: find by ID, or name + + if (count($this->id) === 3) { + Log::debug(sprintf('Finding bill with ID #%d', $this->id['value'])); + /** @var Bill $bill */ + $bill = $this->repository->find(intval($this->id['value'])); + if (!is_null($bill->id)) { + Log::debug(sprintf('Found unmapped bill by ID (#%d): %s', $bill->id, $bill->name)); + + return $bill; + } + Log::debug('Found nothing.'); + } + // 2: find by name + if (count($this->name) === 3) { + /** @var Collection $bills */ + $bills = $this->repository->getBills(); + $name = $this->name['value']; + Log::debug(sprintf('Finding bill with name %s', $name)); + $filtered = $bills->filter( + function (Bill $bill) use ($name) { + if ($bill->name === $name) { + Log::debug(sprintf('Found unmapped bill by name (#%d): %s', $bill->id, $bill->name)); + + return $bill; + } + + return null; + } + ); + + if ($filtered->count() === 1) { + return $filtered->first(); + } + Log::debug('Found nothing.'); + } + + // 4: do not search by account number. + Log::debug('Found NO existing bills.'); + + return new Bill; + + } + + /** + * @return Bill + */ + private function findMappedObject(): Bill + { + Log::debug('In findMappedObject() for Bill'); + $fields = ['id', 'name']; + foreach ($fields as $field) { + $array = $this->$field; + Log::debug(sprintf('Find mapped bill based on field "%s" with value', $field), $array); + // check if a pre-mapped object exists. + $mapped = $this->getMappedObject($array); + if (!is_null($mapped->id)) { + Log::debug(sprintf('Found bill #%d!', $mapped->id)); + + return $mapped; + } + + } + Log::debug('Found no bill on mapped data or no map present.'); + + return new Bill; + } + + /** + * @param array $array + * + * @return Bill + */ + private function getMappedObject(array $array): Bill + { + Log::debug('In getMappedObject() for Bill'); + if (count($array) === 0) { + Log::debug('Array is empty, nothing will come of this.'); + + return new Bill; + } + + if (array_key_exists('mapped', $array) && is_null($array['mapped'])) { + Log::debug(sprintf('No map present for value "%s". Return NULL.', $array['value'])); + + return new Bill; + } + + Log::debug('Finding a mapped bill based on', $array); + + $search = intval($array['mapped']); + $account = $this->repository->find($search); + + Log::debug(sprintf('Found bill! #%d ("%s"). Return it', $account->id, $account->name)); + + return $account; + } + + /** + * @return bool + */ + private function store(): bool + { + // 1: find mapped object: + $mapped = $this->findMappedObject(); + if (!is_null($mapped->id)) { + $this->bill = $mapped; + + return true; + } + // 2: find existing by given values: + $found = $this->findExistingObject(); + if (!is_null($found->id)) { + $this->bill = $found; + + return true; + } + $name = $this->name['value'] ?? ''; + + if (strlen($name) === 0) { + return true; + } + + Log::debug('Found no bill so must create one ourselves.'); + + $data = [ + 'name' => $name, + ]; + + $this->bill = $this->repository->store($data); + Log::debug(sprintf('Successfully stored new bill #%d: %s', $this->bill->id, $this->bill->name)); + + return true; + } } \ No newline at end of file diff --git a/app/Import/Object/ImportJournal.php b/app/Import/Object/ImportJournal.php index 859bd5a2c0..6359852111 100644 --- a/app/Import/Object/ImportJournal.php +++ b/app/Import/Object/ImportJournal.php @@ -33,7 +33,7 @@ class ImportJournal /** @var ImportBudget */ public $budget; /** @var string */ - public $description; + public $description = ''; /** @var Collection */ public $errors; /** @var string */ @@ -43,7 +43,7 @@ class ImportJournal /** @var string */ private $amount = '0'; /** @var ImportBill */ - private $bill; + public $bill; /** @var ImportCategory */ public $category; /** @var ImportCurrency */ @@ -54,11 +54,16 @@ class ImportJournal private $externalId = ''; /** @var array */ private $modifiers = []; + /** @var array */ private $tags = []; + /** @var string */ + public $notes = ''; /** @var string */ private $transactionType = ''; /** @var User */ private $user; + /** @var array */ + public $metaDates = []; /** * ImportEntry constructor. @@ -151,6 +156,7 @@ class ImportJournal $this->opposing->setUser($user); $this->budget->setUser($user); $this->category->setUser($user); + $this->bill->setUser($user); } /** @@ -214,6 +220,12 @@ class ImportJournal case 'description': $this->description = $array['value']; break; + case 'sepa-ct-op': + case 'sepa-ct-id': + case 'sepa-db': + $this->notes .= ' '.$array['value']; + $this->notes = trim($this->notes); + break; case 'external-id': $this->externalId = $array['value']; break; @@ -239,6 +251,16 @@ class ImportJournal case 'tags-space': $this->tags[] = $array; break; + // 'interest_date', 'book_date', 'process_date', 'due_date', 'payment_date', 'invoice_date', + case 'date-interest': + $this->metaDates['interest_date'] = $array['value']; + break; + case 'date-book': + $this->metaDates['book_date'] = $array['value']; + break; + case 'date-process': + $this->metaDates['process_date'] = $array['value']; + break; } } } \ No newline at end of file diff --git a/app/Import/Routine/ImportRoutine.php b/app/Import/Routine/ImportRoutine.php index f881d440a3..9d055adc2c 100644 --- a/app/Import/Routine/ImportRoutine.php +++ b/app/Import/Routine/ImportRoutine.php @@ -21,6 +21,12 @@ use Log; class ImportRoutine { + /** @var Collection */ + public $journals; + /** @var Collection */ + public $errors; + /** @var int */ + public $lines = 0; /** @var ImportJob */ private $job; @@ -31,7 +37,9 @@ class ImportRoutine */ public function __construct(ImportJob $job) { - $this->job = $job; + $this->job = $job; + $this->journals = new Collection; + $this->errors = new Collection; } /** @@ -58,6 +66,9 @@ class ImportRoutine $processor->run(); $objects = $processor->getObjects(); } + $this->lines = $objects->count(); + // once done, use storage thing to actually store them: + Log::debug(sprintf('Returned %d valid objects from file processor', $this->lines)); $storage = new ImportStorage; $storage->setJob($this->job); @@ -65,8 +76,12 @@ class ImportRoutine $storage->setObjects($objects); $storage->store(); - // once done, use storage thing to actually store them: - Log::debug(sprintf('Returned %d valid objects from file processor', $objects->count())); + // update job: + $this->job->status = 'finished'; + $this->job->save(); + + $this->journals = $storage->journals; + $this->errors = $storage->errors; Log::debug(sprintf('Done with import job %s', $this->job->key)); diff --git a/app/Import/Storage/ImportStorage.php b/app/Import/Storage/ImportStorage.php index 3739afbd96..d21f02707b 100644 --- a/app/Import/Storage/ImportStorage.php +++ b/app/Import/Storage/ImportStorage.php @@ -12,15 +12,17 @@ declare(strict_types=1); namespace FireflyIII\Import\Storage; use Amount; -use FireflyIII\Exceptions\FireflyException; +use Carbon\Carbon; use FireflyIII\Import\Object\ImportJournal; use FireflyIII\Models\Account; use FireflyIII\Models\AccountType; use FireflyIII\Models\ImportJob; use FireflyIII\Models\Transaction; +use FireflyIII\Models\TransactionCurrency; use FireflyIII\Models\TransactionJournal; use FireflyIII\Models\TransactionType; use Illuminate\Support\Collection; +use Illuminate\Support\MessageBag; use Log; use Steam; @@ -32,8 +34,12 @@ use Steam; */ class ImportStorage { + /** @var Collection */ + public $errors; + public $journals; /** @var string */ private $dateFormat = 'Ymd'; + /** @var TransactionCurrency */ private $defaultCurrency; /** @var ImportJob */ private $job; @@ -45,7 +51,9 @@ class ImportStorage */ public function __construct() { - $this->objects = new Collection; + $this->objects = new Collection; + $this->journals = new Collection; + $this->errors = new Collection; } /** @@ -87,6 +95,8 @@ class ImportStorage foreach ($this->objects as $index => $object) { Log::debug(sprintf('Going to store object #%d with description "%s"', $index, $object->description)); + $errors = new MessageBag; + // create the asset account $asset = $object->asset->getAccount(); $opposing = new Account; @@ -131,7 +141,10 @@ class ImportStorage $journal->encrypted = 0; $journal->completed = 0; if (!$journal->save()) { - throw new FireflyException($journal->getErrors()->first()); + $errorText = join(', ', $journal->getErrors()->all()); + $errors->add('no-key', sprintf('Error storing journal: %s', $errorText)); + Log::error(sprintf('Could not store line #%d: %s', $index, $errorText)); + continue; } $journal->setMeta('importHash', $object->hash); Log::debug(sprintf('Created journal with ID #%d', $journal->id)); @@ -143,6 +156,10 @@ class ImportStorage $one->transaction_currency_id = $currency->id; $one->amount = $amount; $one->save(); + if (is_null($one->id)) { + $errorText = join(', ', $one->getErrors()->all()); + $errors->add('no-key', sprintf('Error storing transaction one for journal %d: %s', $journal->id, $errorText)); + } Log::debug(sprintf('Created transaction with ID #%d and account #%d', $one->id, $asset->id)); $two = new Transaction; @@ -151,6 +168,10 @@ class ImportStorage $two->transaction_currency_id = $currency->id; $two->amount = Steam::opposite($amount); $two->save(); + if (is_null($two->id)) { + $errorText = join(', ', $two->getErrors()->all()); + $errors->add('no-key', sprintf('Error storing transaction one for journal %d: %s', $journal->id, $errorText)); + } Log::debug(sprintf('Created transaction with ID #%d and account #%d', $two->id, $opposing->id)); // category @@ -167,13 +188,33 @@ class ImportStorage $journal->budgets()->save($budget); } // bill + $bill = $object->bill->getBill(); + if (!is_null($bill->id)) { + Log::debug(sprintf('Linked bill #%d to journal #%d', $bill->id, $journal->id)); + $journal->bill()->associate($bill); + $journal->save(); + } - // - + // all other date fields as meta thing: + foreach ($object->metaDates as $name => $value) { + try { + $date = new Carbon($value); + $journal->setMeta($name, $date); + } catch (\Exception $e) { + // don't care, ignore: + Log::warning(sprintf('Could not parse "%s" into a valid Date object for field %s', $value, $name)); + } + } + // sepa thing as note: + if (strlen($object->notes) > 0) { + $journal->setMeta('notes', $object->notes); + } + $this->journals->push($journal); + $this->errors->push($errors); } - die('Cannot actually store yet.'); + } } \ No newline at end of file diff --git a/app/Models/ImportJob.php b/app/Models/ImportJob.php index 1feb065e20..61338a2fa0 100644 --- a/app/Models/ImportJob.php +++ b/app/Models/ImportJob.php @@ -46,6 +46,7 @@ class ImportJob extends Model 'initialized', 'configured', 'running', + 'finished', ]; /**