mirror of
https://github.com/firefly-iii/firefly-iii.git
synced 2025-02-25 18:45:27 -06:00
Remove unused files in import.
This commit is contained in:
James Cole
parent
091596e80e
commit
c79a577060
@ -1,274 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* ImportEntry.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import;
|
||||
|
||||
use Carbon\Carbon;
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\User;
|
||||
use Illuminate\Support\Collection;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class ImportEntry
|
||||
*
|
||||
* @package FireflyIII\Import
|
||||
*/
|
||||
class ImportEntry
|
||||
{
|
||||
/** @var array */
|
||||
public $certain = [];
|
||||
/** @var Collection */
|
||||
public $errors;
|
||||
/** @var string */
|
||||
public $externalID;
|
||||
/** @var array */
|
||||
public $fields = [];
|
||||
/** @var string */
|
||||
public $hash;
|
||||
/** @var User */
|
||||
public $user;
|
||||
/** @var bool */
|
||||
public $valid = true;
|
||||
/** @var int */
|
||||
private $amountMultiplier = 0;
|
||||
|
||||
/** @var array */
|
||||
private $validFields
|
||||
= ['amount',
|
||||
'date-transaction',
|
||||
'date-interest',
|
||||
'date-book',
|
||||
'description',
|
||||
'date-process',
|
||||
'transaction-type',
|
||||
'currency', 'asset-account', 'opposing-account', 'bill', 'budget', 'category', 'tags'];
|
||||
|
||||
/**
|
||||
* ImportEntry constructor.
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
/** @var string $value */
|
||||
foreach ($this->validFields as $value) {
|
||||
$this->fields[$value] = null;
|
||||
$this->certain[$value] = 0;
|
||||
}
|
||||
$this->errors = new Collection;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $role
|
||||
* @param int $certainty
|
||||
* @param $convertedValue
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function importValue(string $role, int $certainty, $convertedValue)
|
||||
{
|
||||
switch ($role) {
|
||||
default:
|
||||
Log::error('Import entry cannot handle object.', ['role' => $role]);
|
||||
throw new FireflyException('Import entry cannot handle object of type "' . $role . '".');
|
||||
case 'hash':
|
||||
$this->hash = $convertedValue;
|
||||
|
||||
return;
|
||||
case 'amount':
|
||||
/*
|
||||
* Easy enough.
|
||||
*/
|
||||
$this->setFloat('amount', $convertedValue, $certainty);
|
||||
$this->applyMultiplier('amount'); // if present.
|
||||
|
||||
return;
|
||||
case 'account-id':
|
||||
case 'account-iban':
|
||||
case 'account-name':
|
||||
case 'account-number':
|
||||
$this->setObject('asset-account', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'opposing-number':
|
||||
case 'opposing-iban':
|
||||
case 'opposing-id':
|
||||
case 'opposing-name':
|
||||
$this->setObject('opposing-account', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'bill-id':
|
||||
case 'bill-name':
|
||||
$this->setObject('bill', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'budget-id':
|
||||
case 'budget-name':
|
||||
$this->setObject('budget', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'category-id':
|
||||
case 'category-name':
|
||||
$this->setObject('category', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'currency-code':
|
||||
case 'currency-id':
|
||||
case 'currency-name':
|
||||
case 'currency-symbol':
|
||||
$this->setObject('currency', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'date-transaction':
|
||||
$this->setDate('date-transaction', $convertedValue, $certainty);
|
||||
break;
|
||||
|
||||
case 'date-interest':
|
||||
$this->setDate('date-interest', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'date-book':
|
||||
$this->setDate('date-book', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'date-process':
|
||||
$this->setDate('date-process', $convertedValue, $certainty);
|
||||
break;
|
||||
case 'sepa-ct-id':
|
||||
case 'sepa-db':
|
||||
case 'sepa-ct-op':
|
||||
case 'description':
|
||||
$this->setAppendableString('description', $convertedValue);
|
||||
break;
|
||||
case '_ignore':
|
||||
break;
|
||||
case 'ing-debet-credit':
|
||||
case 'rabo-debet-credit':
|
||||
$this->manipulateFloat('amount', 'multiply', $convertedValue);
|
||||
$this->applyMultiplier('amount'); // if present.
|
||||
break;
|
||||
case 'tags-comma':
|
||||
case 'tags-space':
|
||||
$this->appendCollection('tags', $convertedValue);
|
||||
break;
|
||||
case 'external-id':
|
||||
$this->externalID = $convertedValue;
|
||||
break;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param User $user
|
||||
*/
|
||||
public function setUser(User $user)
|
||||
{
|
||||
$this->user = $user;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $field
|
||||
* @param Collection $convertedValue
|
||||
*/
|
||||
private function appendCollection(string $field, Collection $convertedValue)
|
||||
{
|
||||
if (is_null($this->fields[$field])) {
|
||||
$this->fields[$field] = new Collection;
|
||||
}
|
||||
$this->fields[$field] = $this->fields[$field]->merge($convertedValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $field
|
||||
*/
|
||||
private function applyMultiplier(string $field)
|
||||
{
|
||||
if ($this->fields[$field] != 0 && $this->amountMultiplier != 0) {
|
||||
$this->fields[$field] = $this->fields[$field] * $this->amountMultiplier;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $field
|
||||
* @param string $action
|
||||
* @param $convertedValue
|
||||
*
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function manipulateFloat(string $field, string $action, $convertedValue)
|
||||
{
|
||||
switch ($action) {
|
||||
default:
|
||||
Log::error('Cannot handle manipulateFloat', ['field' => $field, 'action' => $action]);
|
||||
throw new FireflyException('Cannot manipulateFloat with action ' . $action);
|
||||
case 'multiply':
|
||||
$this->amountMultiplier = $convertedValue;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $field
|
||||
* @param string $value
|
||||
*/
|
||||
private function setAppendableString(string $field, string $value)
|
||||
{
|
||||
$value = trim($value);
|
||||
$this->fields[$field] .= ' ' . $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $field
|
||||
* @param Carbon $date
|
||||
* @param int $certainty
|
||||
*/
|
||||
private function setDate(string $field, Carbon $date, int $certainty)
|
||||
{
|
||||
if ($certainty > $this->certain[$field] && !is_null($date)) {
|
||||
Log::debug(sprintf('ImportEntry: %s is now %s with certainty %d', $field, $date->format('Y-m-d'), $certainty));
|
||||
$this->fields[$field] = $date;
|
||||
$this->certain[$field] = $certainty;
|
||||
|
||||
return;
|
||||
}
|
||||
Log::info(sprintf('Will not set %s based on certainty %d (current certainty is %d) or NULL id.', $field, $certainty, $this->certain[$field]));
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $field
|
||||
* @param float $value
|
||||
* @param int $certainty
|
||||
*/
|
||||
private function setFloat(string $field, float $value, int $certainty)
|
||||
{
|
||||
if ($certainty > $this->certain[$field]) {
|
||||
Log::debug(sprintf('ImportEntry: %s is now %f with certainty %d', $field, $value, $certainty));
|
||||
$this->fields[$field] = $value;
|
||||
$this->certain[$field] = $certainty;
|
||||
|
||||
return;
|
||||
}
|
||||
Log::info(sprintf('Will not set %s based on certainty %d (current certainty is %d).', $field, $certainty, $this->certain[$field]));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $field
|
||||
* @param $object
|
||||
* @param int $certainty
|
||||
*/
|
||||
private function setObject(string $field, $object, int $certainty)
|
||||
{
|
||||
if ($certainty > $this->certain[$field] && !is_null($object->id)) {
|
||||
Log::debug(sprintf('ImportEntry: %s ID is now %d with certainty %d', $field, $object->id, $certainty));
|
||||
$this->fields[$field] = $object;
|
||||
$this->certain[$field] = $certainty;
|
||||
|
||||
return;
|
||||
}
|
||||
Log::info(sprintf('Will not set %s based on certainty %d (current certainty is %d) or NULL id.', $field, $certainty, $this->certain[$field]));
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* ImportProcedure.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import;
|
||||
|
||||
use FireflyIII\Import\Importer\ImporterInterface;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
|
||||
use Illuminate\Support\Collection;
|
||||
|
||||
/**
|
||||
* Class ImportProcedure
|
||||
*
|
||||
* @package FireflyIII\Import
|
||||
*/
|
||||
class ImportProcedure implements ImportProcedureInterface
|
||||
{
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*
|
||||
* @return Collection
|
||||
*/
|
||||
public function runImport(ImportJob $job): Collection
|
||||
{
|
||||
// update job to say we started.
|
||||
$job->status = 'import_running';
|
||||
$job->save();
|
||||
|
||||
// create Importer
|
||||
$valid = array_keys(config('firefly.import_formats'));
|
||||
$class = 'INVALID';
|
||||
if (in_array($job->file_type, $valid)) {
|
||||
$class = config('firefly.import_formats.' . $job->file_type);
|
||||
}
|
||||
|
||||
/** @var ImporterInterface $importer */
|
||||
$importer = app($class);
|
||||
$importer->setJob($job);
|
||||
|
||||
// create import entries
|
||||
$collection = $importer->createImportEntries();
|
||||
|
||||
// validate / clean collection:
|
||||
$validator = new ImportValidator($collection);
|
||||
$validator->setUser($job->user);
|
||||
$validator->setJob($job);
|
||||
if ($job->configuration['import-account'] != 0) {
|
||||
|
||||
/** @var AccountRepositoryInterface $repository */
|
||||
$repository = app(AccountRepositoryInterface::class);
|
||||
$repository->setUser($job->user);
|
||||
$validator->setDefaultImportAccount($repository->find($job->configuration['import-account']));
|
||||
}
|
||||
|
||||
$cleaned = $validator->clean();
|
||||
|
||||
// then import collection:
|
||||
$storage = new ImportStorage($job->user, $cleaned);
|
||||
$storage->setJob($job);
|
||||
|
||||
// and run store routine:
|
||||
$result = $storage->store();
|
||||
|
||||
// grab import tag:
|
||||
$status = $job->extended_status;
|
||||
$status['importTag'] = $storage->importTag->id;
|
||||
$job->extended_status = $status;
|
||||
$job->status = 'import_complete';
|
||||
$job->save();
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* ImportProcedureInterface.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import;
|
||||
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use Illuminate\Support\Collection;
|
||||
|
||||
/**
|
||||
* Interface ImportProcedureInterface
|
||||
*
|
||||
* @package FireflyIII\Import
|
||||
*/
|
||||
interface ImportProcedureInterface
|
||||
{
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*
|
||||
* @return Collection
|
||||
*/
|
||||
public function runImport(ImportJob $job): Collection;
|
||||
|
||||
}
|
||||
@ -1,420 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* ImportStorage.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import;
|
||||
|
||||
use Carbon\Carbon;
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Models\Rule;
|
||||
use FireflyIII\Models\Tag;
|
||||
use FireflyIII\Models\Transaction;
|
||||
use FireflyIII\Models\TransactionJournal;
|
||||
use FireflyIII\Models\TransactionJournalMeta;
|
||||
use FireflyIII\Models\TransactionType;
|
||||
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
|
||||
use FireflyIII\Rules\Processor;
|
||||
use FireflyIII\User;
|
||||
use Illuminate\Support\Collection;
|
||||
use Log;
|
||||
use Steam;
|
||||
|
||||
/**
|
||||
* Class ImportStorage
|
||||
*
|
||||
* @package FireflyIII\Import
|
||||
*/
|
||||
class ImportStorage
|
||||
{
|
||||
|
||||
/** @var Collection */
|
||||
public $entries;
|
||||
/** @var Tag */
|
||||
public $importTag;
|
||||
/** @var ImportJob */
|
||||
public $job;
|
||||
/** @var User */
|
||||
public $user;
|
||||
|
||||
/** @var Collection */
|
||||
private $rules;
|
||||
|
||||
/**
|
||||
* ImportStorage constructor.
|
||||
*
|
||||
* @param User $user
|
||||
* @param Collection $entries
|
||||
*/
|
||||
public function __construct(User $user, Collection $entries)
|
||||
{
|
||||
$this->entries = $entries;
|
||||
$this->user = $user;
|
||||
$this->rules = $this->getUserRules();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function setJob(ImportJob $job)
|
||||
{
|
||||
$this->job = $job;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param User $user
|
||||
*/
|
||||
public function setUser(User $user)
|
||||
{
|
||||
$this->user = $user;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Collection
|
||||
*/
|
||||
public function store(): Collection
|
||||
{
|
||||
// create a tag to join the transactions.
|
||||
$this->importTag = $this->createImportTag();
|
||||
$collection = new Collection;
|
||||
Log::notice(sprintf('Started storing %d entry(ies).', $this->entries->count()));
|
||||
foreach ($this->entries as $index => $entry) {
|
||||
Log::debug(sprintf('--- import store start for row %d ---', $index));
|
||||
|
||||
// store entry:
|
||||
$journal = $this->storeSingle($index, $entry);
|
||||
$this->job->addStepsDone(1);
|
||||
|
||||
// apply rules:
|
||||
$this->applyRules($journal);
|
||||
$this->job->addStepsDone(1);
|
||||
|
||||
$collection->put($index, $journal);
|
||||
}
|
||||
Log::notice(sprintf('Finished storing %d entry(ies).', $collection->count()));
|
||||
|
||||
return $collection;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $hash
|
||||
*
|
||||
* @return TransactionJournal
|
||||
*/
|
||||
private function alreadyImported(string $hash): TransactionJournal
|
||||
{
|
||||
|
||||
$meta = TransactionJournalMeta
|
||||
::leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
|
||||
->where('journal_meta.name', 'originalImportHash')
|
||||
->where('transaction_journals.user_id', $this->user->id)
|
||||
->where('journal_meta.data', json_encode($hash))->first(['journal_meta.*']);
|
||||
if (!is_null($meta)) {
|
||||
/** @var TransactionJournal $journal */
|
||||
$journal = $meta->transactionjournal;
|
||||
if (intval($journal->completed) === 1) {
|
||||
return $journal;
|
||||
}
|
||||
}
|
||||
|
||||
return new TransactionJournal;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param TransactionJournal $journal
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
private function applyRules(TransactionJournal $journal): bool
|
||||
{
|
||||
if ($this->rules->count() > 0) {
|
||||
|
||||
/** @var Rule $rule */
|
||||
foreach ($this->rules as $rule) {
|
||||
Log::debug(sprintf('Going to apply rule #%d to journal %d.', $rule->id, $journal->id));
|
||||
$processor = Processor::make($rule);
|
||||
$processor->handleTransactionJournal($journal);
|
||||
|
||||
if ($rule->stop_processing) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Tag
|
||||
*/
|
||||
private function createImportTag(): Tag
|
||||
{
|
||||
/** @var TagRepositoryInterface $repository */
|
||||
$repository = app(TagRepositoryInterface::class);
|
||||
$repository->setUser($this->user);
|
||||
$data = [
|
||||
'tag' => trans('firefly.import_with_key', ['key' => $this->job->key]),
|
||||
'date' => new Carbon,
|
||||
'description' => null,
|
||||
'latitude' => null,
|
||||
'longitude' => null,
|
||||
'zoomLevel' => null,
|
||||
'tagMode' => 'nothing',
|
||||
];
|
||||
$tag = $repository->store($data);
|
||||
|
||||
return $tag;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Collection
|
||||
*/
|
||||
private function getUserRules(): Collection
|
||||
{
|
||||
$set = Rule::distinct()
|
||||
->where('rules.user_id', $this->user->id)
|
||||
->leftJoin('rule_groups', 'rule_groups.id', '=', 'rules.rule_group_id')
|
||||
->leftJoin('rule_triggers', 'rules.id', '=', 'rule_triggers.rule_id')
|
||||
->where('rule_groups.active', 1)
|
||||
->where('rule_triggers.trigger_type', 'user_action')
|
||||
->where('rule_triggers.trigger_value', 'store-journal')
|
||||
->where('rules.active', 1)
|
||||
->orderBy('rule_groups.order', 'ASC')
|
||||
->orderBy('rules.order', 'ASC')
|
||||
->get(['rules.*', 'rule_groups.order']);
|
||||
Log::debug(sprintf('Found %d user rules.', $set->count()));
|
||||
|
||||
return $set;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $entry
|
||||
*
|
||||
* @return array
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function storeAccounts($entry): array
|
||||
{
|
||||
// then create transactions. Single ones, unfortunately.
|
||||
switch ($entry->fields['transaction-type']->type) {
|
||||
default:
|
||||
throw new FireflyException('ImportStorage cannot handle ' . $entry->fields['transaction-type']->type);
|
||||
case TransactionType::WITHDRAWAL:
|
||||
$source = $entry->fields['asset-account'];
|
||||
$destination = $entry->fields['opposing-account'];
|
||||
// make amount positive, if it is not.
|
||||
break;
|
||||
case TransactionType::DEPOSIT:
|
||||
$source = $entry->fields['opposing-account'];
|
||||
$destination = $entry->fields['asset-account'];
|
||||
break;
|
||||
case TransactionType::TRANSFER:
|
||||
// depends on amount:
|
||||
if ($entry->fields['amount'] < 0) {
|
||||
$source = $entry->fields['asset-account'];
|
||||
$destination = $entry->fields['opposing-account'];
|
||||
break;
|
||||
}
|
||||
$destination = $entry->fields['asset-account'];
|
||||
$source = $entry->fields['opposing-account'];
|
||||
break;
|
||||
}
|
||||
|
||||
return [
|
||||
'source' => $source,
|
||||
'destination' => $destination,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param TransactionJournal $journal
|
||||
* @param ImportEntry $entry
|
||||
*/
|
||||
private function storeBill(TransactionJournal $journal, ImportEntry $entry)
|
||||
{
|
||||
|
||||
if (!is_null($entry->fields['bill']) && !is_null($entry->fields['bill']->id)) {
|
||||
$journal->bill()->associate($entry->fields['bill']);
|
||||
Log::debug('Attached bill', ['id' => $entry->fields['bill']->id, 'name' => $entry->fields['bill']->name]);
|
||||
$journal->save();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param TransactionJournal $journal
|
||||
* @param ImportEntry $entry
|
||||
*/
|
||||
private function storeBudget(TransactionJournal $journal, ImportEntry $entry)
|
||||
{
|
||||
if (!is_null($entry->fields['budget']) && !is_null($entry->fields['budget']->id)) {
|
||||
$journal->budgets()->save($entry->fields['budget']);
|
||||
Log::debug('Attached budget', ['id' => $entry->fields['budget']->id, 'name' => $entry->fields['budget']->name]);
|
||||
$journal->save();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param TransactionJournal $journal
|
||||
* @param ImportEntry $entry
|
||||
*/
|
||||
private function storeCategory(TransactionJournal $journal, ImportEntry $entry)
|
||||
{
|
||||
if (!is_null($entry->fields['category']) && !is_null($entry->fields['category']->id)) {
|
||||
$journal->categories()->save($entry->fields['category']);
|
||||
Log::debug('Attached category', ['id' => $entry->fields['category']->id, 'name' => $entry->fields['category']->name]);
|
||||
$journal->save();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $entry
|
||||
*
|
||||
* @return TransactionJournal
|
||||
*/
|
||||
private function storeJournal($entry): TransactionJournal
|
||||
{
|
||||
|
||||
$billId = is_null($entry->fields['bill']) || intval($entry->fields['bill']->id) === 0 ? null : intval($entry->fields['bill']->id);
|
||||
$journalData = [
|
||||
'user_id' => $entry->user->id,
|
||||
'transaction_type_id' => $entry->fields['transaction-type']->id,
|
||||
'bill_id' => $billId,
|
||||
'transaction_currency_id' => $entry->fields['currency']->id,
|
||||
'description' => $entry->fields['description'],
|
||||
'date' => $entry->fields['date-transaction'],
|
||||
'interest_date' => $entry->fields['date-interest'],
|
||||
'book_date' => $entry->fields['date-book'],
|
||||
'process_date' => $entry->fields['date-process'],
|
||||
'completed' => 0,
|
||||
];
|
||||
/** @var TransactionJournal $journal */
|
||||
$journal = TransactionJournal::create($journalData);
|
||||
|
||||
foreach ($journal->getErrors()->all() as $err) {
|
||||
Log::error('Error when storing journal: ' . $err);
|
||||
}
|
||||
Log::debug('Created journal', ['id' => $journal->id]);
|
||||
|
||||
// save hash as meta value:
|
||||
$meta = new TransactionJournalMeta;
|
||||
$meta->name = 'originalImportHash';
|
||||
$meta->data = $entry->hash;
|
||||
$meta->transactionJournal()->associate($journal);
|
||||
$meta->save();
|
||||
|
||||
return $journal;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $index
|
||||
* @param ImportEntry $entry
|
||||
*
|
||||
* @return TransactionJournal
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function storeSingle(int $index, ImportEntry $entry): TransactionJournal
|
||||
{
|
||||
if ($entry->valid === false) {
|
||||
Log::warning(sprintf('Cannot import row %d, because the entry is not valid.', $index));
|
||||
$errors = join(', ', $entry->errors->all());
|
||||
$errorText = sprintf('Row #%d: ' . $errors, $index);
|
||||
$extendedStatus = $this->job->extended_status;
|
||||
$extendedStatus['errors'][] = $errorText;
|
||||
$this->job->extended_status = $extendedStatus;
|
||||
$this->job->save();
|
||||
|
||||
return new TransactionJournal;
|
||||
}
|
||||
$alreadyImported = $this->alreadyImported($entry->hash);
|
||||
if (!is_null($alreadyImported->id)) {
|
||||
Log::warning(sprintf('Cannot import row %d, because it has already been imported (journal #%d).', $index, $alreadyImported->id));
|
||||
$errorText = trans(
|
||||
'firefly.import_double',
|
||||
['row' => $index, 'link' => route('transactions.show', [$alreadyImported->id]), 'description' => $alreadyImported->description]
|
||||
);
|
||||
$extendedStatus = $this->job->extended_status;
|
||||
$extendedStatus['errors'][] = $errorText;
|
||||
$this->job->extended_status = $extendedStatus;
|
||||
$this->job->save();
|
||||
|
||||
return new TransactionJournal;
|
||||
}
|
||||
|
||||
Log::debug(sprintf('Going to store row %d', $index));
|
||||
|
||||
|
||||
$journal = $this->storeJournal($entry);
|
||||
$amount = Steam::positive($entry->fields['amount']);
|
||||
$accounts = $this->storeAccounts($entry);
|
||||
|
||||
// create new transactions. This is something that needs a rewrite for multiple/split transactions.
|
||||
$sourceData = [
|
||||
'account_id' => $accounts['source']->id,
|
||||
'transaction_journal_id' => $journal->id,
|
||||
'transaction_currency_id' => $journal->transaction_currency_id,
|
||||
'description' => null,
|
||||
'amount' => bcmul($amount, '-1'),
|
||||
];
|
||||
|
||||
$destinationData = [
|
||||
'account_id' => $accounts['destination']->id,
|
||||
'transaction_currency_id' => $journal->transaction_currency_id,
|
||||
'transaction_journal_id' => $journal->id,
|
||||
'description' => null,
|
||||
'amount' => $amount,
|
||||
];
|
||||
|
||||
$one = Transaction::create($sourceData);
|
||||
$two = Transaction::create($destinationData);
|
||||
$error = false;
|
||||
if (is_null($one->id)) {
|
||||
Log::error('Could not create transaction 1.', $one->getErrors()->all());
|
||||
$error = true;
|
||||
}
|
||||
|
||||
if (is_null($two->id)) {
|
||||
Log::error('Could not create transaction 2.', $two->getErrors()->all());
|
||||
$error = true;
|
||||
}
|
||||
|
||||
// respond to error
|
||||
if ($error === true) {
|
||||
$errorText = sprintf('Cannot import row %d, because an error occured when storing data.', $index);
|
||||
Log::error($errorText);
|
||||
$extendedStatus = $this->job->extended_status;
|
||||
$extendedStatus['errors'][] = $errorText;
|
||||
$this->job->extended_status = $extendedStatus;
|
||||
$this->job->save();
|
||||
|
||||
return new TransactionJournal;
|
||||
}
|
||||
|
||||
Log::debug('Created transaction 1', ['id' => $one->id, 'account' => $one->account_id, 'account_name' => $accounts['source']->name]);
|
||||
Log::debug('Created transaction 2', ['id' => $two->id, 'account' => $two->account_id, 'account_name' => $accounts['destination']->name]);
|
||||
|
||||
$journal->completed = 1;
|
||||
$journal->save();
|
||||
|
||||
// attach import tag.
|
||||
$journal->tags()->save($this->importTag);
|
||||
|
||||
// now attach budget and so on.
|
||||
$this->storeBudget($journal, $entry);
|
||||
$this->storeCategory($journal, $entry);
|
||||
$this->storeBill($journal, $entry);
|
||||
|
||||
return $journal;
|
||||
}
|
||||
}
|
||||
@ -1,438 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* ImportValidator.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import;
|
||||
|
||||
use Carbon\Carbon;
|
||||
use FireflyIII\Models\Account;
|
||||
use FireflyIII\Models\AccountType;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Models\TransactionType;
|
||||
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
|
||||
use FireflyIII\Repositories\Currency\CurrencyRepositoryInterface;
|
||||
use FireflyIII\User;
|
||||
use Illuminate\Support\Collection;
|
||||
use Log;
|
||||
use Preferences;
|
||||
|
||||
/**
|
||||
* Class ImportValidator
|
||||
*
|
||||
* @package FireflyIII\Import
|
||||
*/
|
||||
class ImportValidator
|
||||
{
|
||||
/** @var ImportJob */
|
||||
public $job;
|
||||
/** @var Account */
|
||||
protected $defaultImportAccount;
|
||||
/** @var Collection */
|
||||
protected $entries;
|
||||
/** @var User */
|
||||
protected $user;
|
||||
|
||||
/**
|
||||
* ImportValidator constructor.
|
||||
*
|
||||
* @param Collection $entries
|
||||
*/
|
||||
public function __construct(Collection $entries)
|
||||
{
|
||||
$this->entries = $entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean collection by filling in all the blanks.
|
||||
*/
|
||||
public function clean(): Collection
|
||||
{
|
||||
Log::notice(sprintf('Started validating %d entry(ies).', $this->entries->count()));
|
||||
$newCollection = new Collection;
|
||||
/** @var ImportEntry $entry */
|
||||
foreach ($this->entries as $index => $entry) {
|
||||
Log::debug(sprintf('--- import validator start for row %d ---', $index));
|
||||
/*
|
||||
* X Adds the date (today) if no date is present.
|
||||
* X Determins the types of accounts involved (asset, expense, revenue).
|
||||
* X Determins the type of transaction (withdrawal, deposit, transfer).
|
||||
* - Determins the currency of the transaction.
|
||||
* X Adds a default description if there isn't one present.
|
||||
*/
|
||||
$entry = $this->checkAmount($entry);
|
||||
$entry = $this->setDate($entry);
|
||||
$entry = $this->setAssetAccount($entry);
|
||||
$entry = $this->setOpposingAccount($entry);
|
||||
$entry = $this->cleanDescription($entry);
|
||||
$entry = $this->setTransactionType($entry);
|
||||
$entry = $this->setTransactionCurrency($entry);
|
||||
|
||||
$newCollection->put($index, $entry);
|
||||
$this->job->addStepsDone(1);
|
||||
}
|
||||
Log::notice(sprintf('Finished validating %d entry(ies).', $newCollection->count()));
|
||||
|
||||
return $newCollection;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Account $defaultImportAccount
|
||||
*/
|
||||
public function setDefaultImportAccount(Account $defaultImportAccount)
|
||||
{
|
||||
$this->defaultImportAccount = $defaultImportAccount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function setJob(ImportJob $job)
|
||||
{
|
||||
$this->job = $job;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param User $user
|
||||
*/
|
||||
public function setUser(User $user)
|
||||
{
|
||||
$this->user = $user;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportEntry $entry
|
||||
*
|
||||
* @return ImportEntry
|
||||
*/
|
||||
private function checkAmount(ImportEntry $entry): ImportEntry
|
||||
{
|
||||
if ($entry->fields['amount'] == 0) {
|
||||
$entry->valid = false;
|
||||
$entry->errors->push('Amount of transaction is zero, cannot handle.');
|
||||
Log::warning('Amount of transaction is zero, cannot handle.');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
Log::debug('Amount is OK.');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportEntry $entry
|
||||
*
|
||||
* @return ImportEntry
|
||||
*/
|
||||
private function cleanDescription(ImportEntry $entry): ImportEntry
|
||||
{
|
||||
|
||||
if (!isset($entry->fields['description'])) {
|
||||
Log::debug('Set empty transaction description because field was not set.');
|
||||
$entry->fields['description'] = '(empty transaction description)';
|
||||
|
||||
return $entry;
|
||||
}
|
||||
if (is_null($entry->fields['description'])) {
|
||||
Log::debug('Set empty transaction description because field was null.');
|
||||
$entry->fields['description'] = '(empty transaction description)';
|
||||
|
||||
return $entry;
|
||||
}
|
||||
$entry->fields['description'] = trim($entry->fields['description']);
|
||||
|
||||
if (strlen($entry->fields['description']) == 0) {
|
||||
Log::debug('Set empty transaction description because field was empty.');
|
||||
$entry->fields['description'] = '(empty transaction description)';
|
||||
|
||||
return $entry;
|
||||
}
|
||||
Log::debug('Transaction description is OK.', ['description' => $entry->fields['description']]);
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Account $account
|
||||
* @param string $type
|
||||
*
|
||||
* @return Account
|
||||
*/
|
||||
private function convertAccount(Account $account, string $type): Account
|
||||
{
|
||||
$accountType = $account->accountType->type;
|
||||
if ($accountType === $type) {
|
||||
Log::debug(sprintf('Account %s already of type %s', $account->name, $type));
|
||||
|
||||
return $account;
|
||||
}
|
||||
// find it first by new type:
|
||||
|
||||
/** @var AccountRepositoryInterface $repository */
|
||||
$repository = app(AccountRepositoryInterface::class);
|
||||
$repository->setUser($this->user);
|
||||
|
||||
$result = $repository->findByName($account->name, [$type]);
|
||||
if (is_null($result->id)) {
|
||||
// can convert account:
|
||||
Log::debug(sprintf('No account named %s of type %s, create new account.', $account->name, $type));
|
||||
$result = $repository->store(
|
||||
[
|
||||
'user' => $this->user->id,
|
||||
'accountType' => config('firefly.shortNamesByFullName.' . $type),
|
||||
'name' => $account->name,
|
||||
'virtualBalance' => 0,
|
||||
'active' => true,
|
||||
'iban' => null,
|
||||
'openingBalance' => 0,
|
||||
]
|
||||
);
|
||||
}
|
||||
Log::debug(
|
||||
sprintf(
|
||||
'Using another account named %s (#%d) of type %s, will use that one instead of %s (#%d)', $account->name, $result->id, $type, $account->name,
|
||||
$account->id
|
||||
)
|
||||
);
|
||||
|
||||
return $result;
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Account
|
||||
*/
|
||||
private function fallbackExpenseAccount(): Account
|
||||
{
|
||||
|
||||
/** @var AccountRepositoryInterface $repository */
|
||||
$repository = app(AccountRepositoryInterface::class);
|
||||
$repository->setUser($this->user);
|
||||
|
||||
$name = 'Unknown expense account';
|
||||
$result = $repository->findByName($name, [AccountType::EXPENSE]);
|
||||
if (is_null($result->id)) {
|
||||
$result = $repository->store(
|
||||
['name' => $name, 'iban' => null, 'openingBalance' => 0, 'user' => $this->user->id, 'accountType' => 'expense', 'virtualBalance' => 0,
|
||||
'active' => true]
|
||||
);
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Account
|
||||
*/
|
||||
private function fallbackRevenueAccount(): Account
|
||||
{
|
||||
|
||||
/** @var AccountRepositoryInterface $repository */
|
||||
$repository = app(AccountRepositoryInterface::class);
|
||||
$repository->setUser($this->user);
|
||||
|
||||
$name = 'Unknown revenue account';
|
||||
$result = $repository->findByName($name, [AccountType::REVENUE]);
|
||||
|
||||
|
||||
if (is_null($result->id)) {
|
||||
$result = $repository->store(
|
||||
['name' => $name, 'iban' => null, 'openingBalance' => 0, 'user' => $this->user->id, 'accountType' => 'revenue', 'virtualBalance' => 0,
|
||||
'active' => true]
|
||||
);
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportEntry $entry
|
||||
*
|
||||
* @return ImportEntry
|
||||
*/
|
||||
private function setAssetAccount(ImportEntry $entry): ImportEntry
|
||||
{
|
||||
if (is_null($entry->fields['asset-account'])) {
|
||||
if (!is_null($this->defaultImportAccount)) {
|
||||
Log::debug('Set asset account from default asset account');
|
||||
$entry->fields['asset-account'] = $this->defaultImportAccount;
|
||||
|
||||
return $entry;
|
||||
}
|
||||
// default import is null? should not happen. Entry cannot be imported.
|
||||
// set error message and block.
|
||||
$entry->valid = false;
|
||||
Log::warning('Cannot import entry. Asset account is NULL and import account is also NULL.');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
Log::debug('Asset account is OK.', ['id' => $entry->fields['asset-account']->id, 'name' => $entry->fields['asset-account']->name]);
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param ImportEntry $entry
|
||||
*
|
||||
* @return ImportEntry
|
||||
*/
|
||||
private function setDate(ImportEntry $entry): ImportEntry
|
||||
{
|
||||
if (is_null($entry->fields['date-transaction']) || $entry->certain['date-transaction'] == 0) {
|
||||
// empty date field? find alternative.
|
||||
$alternatives = ['date-book', 'date-interest', 'date-process'];
|
||||
foreach ($alternatives as $alternative) {
|
||||
if (!is_null($entry->fields[$alternative])) {
|
||||
Log::debug(sprintf('Copied date-transaction from %s.', $alternative));
|
||||
$entry->fields['date-transaction'] = clone $entry->fields[$alternative];
|
||||
|
||||
return $entry;
|
||||
}
|
||||
}
|
||||
// date is still null at this point
|
||||
Log::debug('Set date-transaction to today.');
|
||||
$entry->fields['date-transaction'] = new Carbon;
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
// confidence is zero?
|
||||
|
||||
Log::debug('Date-transaction is OK');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportEntry $entry
|
||||
*
|
||||
* @return ImportEntry
|
||||
*/
|
||||
private function setOpposingAccount(ImportEntry $entry): ImportEntry
|
||||
{
|
||||
// empty opposing account. Create one based on amount.
|
||||
if (is_null($entry->fields['opposing-account'])) {
|
||||
|
||||
if ($entry->fields['amount'] < 0) {
|
||||
// create or find general opposing expense account.
|
||||
Log::debug('Created fallback expense account');
|
||||
$entry->fields['opposing-account'] = $this->fallbackExpenseAccount();
|
||||
|
||||
return $entry;
|
||||
}
|
||||
Log::debug('Created fallback revenue account');
|
||||
$entry->fields['opposing-account'] = $this->fallbackRevenueAccount();
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
// opposing is of type "import". Convert to correct type (by amount):
|
||||
$type = $entry->fields['opposing-account']->accountType->type;
|
||||
if ($type == AccountType::IMPORT && $entry->fields['amount'] < 0) {
|
||||
$account = $this->convertAccount($entry->fields['opposing-account'], AccountType::EXPENSE);
|
||||
$entry->fields['opposing-account'] = $account;
|
||||
Log::debug('Converted import account to expense account');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
if ($type == AccountType::IMPORT && $entry->fields['amount'] > 0) {
|
||||
$account = $this->convertAccount($entry->fields['opposing-account'], AccountType::REVENUE);
|
||||
$entry->fields['opposing-account'] = $account;
|
||||
Log::debug('Converted import account to revenue account');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
// amount < 0, but opposing is revenue
|
||||
if ($type == AccountType::REVENUE && $entry->fields['amount'] < 0) {
|
||||
$account = $this->convertAccount($entry->fields['opposing-account'], AccountType::EXPENSE);
|
||||
$entry->fields['opposing-account'] = $account;
|
||||
Log::debug('Converted revenue account to expense account');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
// amount > 0, but opposing is expense
|
||||
if ($type == AccountType::EXPENSE && $entry->fields['amount'] > 0) {
|
||||
$account = $this->convertAccount($entry->fields['opposing-account'], AccountType::REVENUE);
|
||||
$entry->fields['opposing-account'] = $account;
|
||||
Log::debug('Converted expense account to revenue account');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
// account type is OK
|
||||
Log::debug('Opposing account is OK.');
|
||||
|
||||
return $entry;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportEntry $entry
|
||||
*
|
||||
* @return ImportEntry
|
||||
*/
|
||||
private function setTransactionCurrency(ImportEntry $entry): ImportEntry
|
||||
{
|
||||
if (is_null($entry->fields['currency'])) {
|
||||
/** @var CurrencyRepositoryInterface $repository */
|
||||
$repository = app(CurrencyRepositoryInterface::class);
|
||||
$repository->setUser($this->user);
|
||||
// is the default currency for the user or the system
|
||||
$defaultCode = Preferences::getForUser($this->user, 'currencyPreference', config('firefly.default_currency', 'EUR'))->data;
|
||||
|
||||
$entry->fields['currency'] = $repository->findByCode($defaultCode);
|
||||
Log::debug(sprintf('Set currency to %s', $defaultCode));
|
||||
|
||||
return $entry;
|
||||
}
|
||||
Log::debug(sprintf('Currency is OK: %s', $entry->fields['currency']->code));
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportEntry $entry
|
||||
*
|
||||
* @return ImportEntry
|
||||
*/
|
||||
private function setTransactionType(ImportEntry $entry): ImportEntry
|
||||
{
|
||||
Log::debug(sprintf('Opposing account is of type %s', $entry->fields['opposing-account']->accountType->type));
|
||||
$type = $entry->fields['opposing-account']->accountType->type;
|
||||
switch ($type) {
|
||||
case AccountType::EXPENSE:
|
||||
$entry->fields['transaction-type'] = TransactionType::whereType(TransactionType::WITHDRAWAL)->first();
|
||||
Log::debug('Transaction type is now withdrawal.');
|
||||
|
||||
return $entry;
|
||||
case AccountType::REVENUE:
|
||||
$entry->fields['transaction-type'] = TransactionType::whereType(TransactionType::DEPOSIT)->first();
|
||||
Log::debug('Transaction type is now deposit.');
|
||||
|
||||
return $entry;
|
||||
case AccountType::DEFAULT:
|
||||
case AccountType::ASSET:
|
||||
$entry->fields['transaction-type'] = TransactionType::whereType(TransactionType::TRANSFER)->first();
|
||||
Log::debug('Transaction type is now transfer.');
|
||||
|
||||
return $entry;
|
||||
}
|
||||
Log::warning(sprintf('Opposing account is of type %s, cannot handle this.', $type));
|
||||
$entry->valid = false;
|
||||
$entry->errors->push(sprintf('Opposing account is of type %s, cannot handle this.', $type));
|
||||
|
||||
return $entry;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,175 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* CsvImporter.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import\Importer;
|
||||
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Import\Converter\ConverterInterface;
|
||||
use FireflyIII\Import\ImportEntry;
|
||||
use FireflyIII\Import\Specifics\SpecificInterface;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use Illuminate\Support\Collection;
|
||||
use League\Csv\Reader;
|
||||
use Log;
|
||||
|
||||
/**
|
||||
* Class CsvImporter
|
||||
*
|
||||
* @package FireflyIII\Import\Importer
|
||||
*/
|
||||
class CsvImporter implements ImporterInterface
|
||||
{
|
||||
/** @var Collection */
|
||||
public $collection;
|
||||
/** @var ImportJob */
|
||||
public $job;
|
||||
|
||||
public $validSpecifics = [];
|
||||
|
||||
/**
|
||||
* CsvImporter constructor.
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->collection = new Collection;
|
||||
$this->validSpecifics = array_keys(config('csv.import_specifics'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the actual import
|
||||
*
|
||||
* @return Collection
|
||||
*/
|
||||
public function createImportEntries(): Collection
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
$content = $this->job->uploadFileContents();
|
||||
|
||||
// create CSV reader.
|
||||
$reader = Reader::createFromString($content);
|
||||
$reader->setDelimiter($config['delimiter']);
|
||||
$start = $config['has-headers'] ? 1 : 0;
|
||||
$results = $reader->fetch();
|
||||
|
||||
Log::notice('Building importable objects from CSV file.');
|
||||
|
||||
foreach ($results as $index => $row) {
|
||||
if ($index >= $start) {
|
||||
$line = $index + 1;
|
||||
Log::debug('----- import entry build start --');
|
||||
Log::debug(sprintf('Now going to import row %d.', $index));
|
||||
$importEntry = $this->importSingleRow($index, $row);
|
||||
$this->collection->put($line, $importEntry);
|
||||
/**
|
||||
* 1. Build import entry.
|
||||
* 2. Validate import entry.
|
||||
* 3. Store journal.
|
||||
* 4. Run rules.
|
||||
*/
|
||||
$this->job->addTotalSteps(4);
|
||||
$this->job->addStepsDone(1);
|
||||
}
|
||||
}
|
||||
Log::debug(sprintf('Import collection contains %d entries', $this->collection->count()));
|
||||
Log::notice(sprintf('Built %d importable object(s) from your CSV file.', $this->collection->count()));
|
||||
|
||||
return $this->collection;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function setJob(ImportJob $job)
|
||||
{
|
||||
$this->job = $job;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $index
|
||||
* @param array $row
|
||||
*
|
||||
* @return ImportEntry
|
||||
* @throws FireflyException
|
||||
*/
|
||||
private function importSingleRow(int $index, array $row): ImportEntry
|
||||
{
|
||||
// create import object. This is where each entry ends up.
|
||||
$object = new ImportEntry;
|
||||
|
||||
Log::debug(sprintf('Now at row %d', $index));
|
||||
|
||||
// set some vars:
|
||||
$object->setUser($this->job->user);
|
||||
$config = $this->job->configuration;
|
||||
$json = json_encode($row);
|
||||
|
||||
if ($json === false) {
|
||||
throw new FireflyException(sprintf('Could not process row #%d. Are you sure the uploaded file is encoded as "UTF-8"?', $index));
|
||||
}
|
||||
|
||||
// hash the row:
|
||||
$hash = hash('sha256', $json);
|
||||
$object->importValue('hash', 100, $hash);
|
||||
|
||||
// and this is the point where the specifix go to work.
|
||||
foreach ($config['specifics'] as $name => $enabled) {
|
||||
|
||||
if (!in_array($name, $this->validSpecifics)) {
|
||||
throw new FireflyException(sprintf('"%s" is not a valid class name', $name));
|
||||
}
|
||||
|
||||
/** @var SpecificInterface $specific */
|
||||
$specific = app('FireflyIII\Import\Specifics\\' . $name);
|
||||
|
||||
// it returns the row, possibly modified:
|
||||
$row = $specific->run($row);
|
||||
}
|
||||
|
||||
foreach ($row as $rowIndex => $value) {
|
||||
// find the role for this column:
|
||||
$role = $config['column-roles'][$rowIndex] ?? '_ignore';
|
||||
$doMap = $config['column-do-mapping'][$rowIndex] ?? false;
|
||||
$validConverters = array_keys(config('csv.import_roles'));
|
||||
|
||||
// throw error when not a valid converter.
|
||||
if (!in_array($role, $validConverters)) {
|
||||
throw new FireflyException(sprintf('"%s" is not a valid role.', $role));
|
||||
}
|
||||
$converterClass = config('csv.import_roles.' . $role . '.converter');
|
||||
$mapping = $config['column-mapping-config'][$rowIndex] ?? [];
|
||||
$className = sprintf('FireflyIII\\Import\\Converter\\%s', $converterClass);
|
||||
/** @var ConverterInterface $converter */
|
||||
$converter = app($className);
|
||||
// set some useful values for the converter:
|
||||
$converter->setMapping($mapping);
|
||||
$converter->setDoMap($doMap);
|
||||
$converter->setUser($this->job->user);
|
||||
$converter->setConfig($config);
|
||||
|
||||
// run the converter for this value:
|
||||
$convertedValue = $converter->convert($value);
|
||||
$certainty = $converter->getCertainty();
|
||||
|
||||
// log it.
|
||||
Log::debug('Value ', ['index' => $rowIndex, 'value' => $value, 'role' => $role]);
|
||||
|
||||
// store in import entry:
|
||||
Log::debug('Going to import', ['role' => $role, 'value' => $value, 'certainty' => $certainty]);
|
||||
$object->importValue($role, $certainty, $convertedValue);
|
||||
}
|
||||
|
||||
|
||||
return $object;
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* ImporterInterface.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import\Importer;
|
||||
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use Illuminate\Support\Collection;
|
||||
|
||||
/**
|
||||
* Interface ImporterInterface
|
||||
*
|
||||
* @package FireflyIII\Import\Importer
|
||||
*/
|
||||
interface ImporterInterface
|
||||
{
|
||||
/**
|
||||
* Run the actual import
|
||||
*
|
||||
* @return Collection
|
||||
*/
|
||||
public function createImportEntries(): Collection;
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*
|
||||
*/
|
||||
public function setJob(ImportJob $job);
|
||||
}
|
||||
@ -1,286 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* CsvSetup.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import\Setup;
|
||||
|
||||
|
||||
use ExpandedForm;
|
||||
use FireflyIII\Exceptions\FireflyException;
|
||||
use FireflyIII\Models\Account;
|
||||
use FireflyIII\Models\AccountType;
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use FireflyIII\Repositories\Account\AccountRepositoryInterface;
|
||||
use FireflyIII\Support\Import\CsvImportSupportTrait;
|
||||
use Illuminate\Http\Request;
|
||||
use Log;
|
||||
use Symfony\Component\HttpFoundation\FileBag;
|
||||
|
||||
/**
|
||||
* Class CsvSetup
|
||||
*
|
||||
* @package FireflyIII\Import\Importer
|
||||
*/
|
||||
class CsvSetup implements SetupInterface
|
||||
{
|
||||
use CsvImportSupportTrait;
|
||||
/** @var Account */
|
||||
public $defaultImportAccount;
|
||||
/** @var ImportJob */
|
||||
public $job;
|
||||
|
||||
/**
|
||||
* CsvImporter constructor.
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->defaultImportAccount = new Account;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create initial (empty) configuration array.
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function configure(): bool
|
||||
{
|
||||
if (is_null($this->job->configuration) || (is_array($this->job->configuration) && count($this->job->configuration) === 0)) {
|
||||
Log::debug('No config detected, will create empty one.');
|
||||
$this->job->configuration = config('csv.default_config');
|
||||
$this->job->save();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// need to do nothing, for now.
|
||||
Log::debug('Detected config in upload, will use that one. ', $this->job->configuration);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
public function getConfigurationData(): array
|
||||
{
|
||||
/** @var AccountRepositoryInterface $accountRepository */
|
||||
$accountRepository = app(AccountRepositoryInterface::class);
|
||||
$accounts = $accountRepository->getAccountsByType([AccountType::DEFAULT, AccountType::ASSET]);
|
||||
$delimiters = [
|
||||
',' => trans('form.csv_comma'),
|
||||
';' => trans('form.csv_semicolon'),
|
||||
'tab' => trans('form.csv_tab'),
|
||||
];
|
||||
|
||||
$specifics = [];
|
||||
|
||||
// collect specifics.
|
||||
foreach (config('csv.import_specifics') as $name => $className) {
|
||||
$specifics[$name] = [
|
||||
'name' => $className::getName(),
|
||||
'description' => $className::getDescription(),
|
||||
];
|
||||
}
|
||||
|
||||
$data = [
|
||||
'accounts' => ExpandedForm::makeSelectList($accounts),
|
||||
'specifix' => [],
|
||||
'delimiters' => $delimiters,
|
||||
'upload_path' => storage_path('upload'),
|
||||
'is_upload_possible' => is_writable(storage_path('upload')),
|
||||
'specifics' => $specifics,
|
||||
];
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the data required for the view that will let the user add settings to the import job.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getDataForSettings(): array
|
||||
{
|
||||
Log::debug('Now in getDataForSettings()');
|
||||
if ($this->doColumnRoles()) {
|
||||
Log::debug('doColumnRoles() is true.');
|
||||
$data = $this->getDataForColumnRoles();
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
if ($this->doColumnMapping()) {
|
||||
Log::debug('doColumnMapping() is true.');
|
||||
$data = $this->getDataForColumnMapping();
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
echo 'no settings to do.';
|
||||
exit;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the name of the view that will be shown to the user to further configure
|
||||
* the import job.
|
||||
*
|
||||
* @return string
|
||||
* @throws FireflyException
|
||||
*/
|
||||
public function getViewForSettings(): string
|
||||
{
|
||||
if ($this->doColumnRoles()) {
|
||||
return 'import.csv.roles';
|
||||
}
|
||||
|
||||
if ($this->doColumnMapping()) {
|
||||
return 'import.csv.map';
|
||||
}
|
||||
throw new FireflyException('There is no view for the current CSV import step.');
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns whether or not the user must configure this import
|
||||
* job further.
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function requireUserSettings(): bool
|
||||
{
|
||||
Log::debug(sprintf('doColumnMapping is %s', $this->doColumnMapping()));
|
||||
Log::debug(sprintf('doColumnRoles is %s', $this->doColumnRoles()));
|
||||
if ($this->doColumnMapping() || $this->doColumnRoles()) {
|
||||
Log::debug('Return true');
|
||||
|
||||
return true;
|
||||
}
|
||||
Log::debug('Return false');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $data
|
||||
*
|
||||
* @param FileBag $files
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function saveImportConfiguration(array $data, FileBag $files): bool
|
||||
{
|
||||
/** @var AccountRepositoryInterface $repository */
|
||||
$repository = app(AccountRepositoryInterface::class);
|
||||
$importId = $data['csv_import_account'] ?? 0;
|
||||
$account = $repository->find(intval($importId));
|
||||
|
||||
$hasHeaders = isset($data['has_headers']) && intval($data['has_headers']) === 1 ? true : false;
|
||||
$config = $this->job->configuration;
|
||||
$config['has-headers'] = $hasHeaders;
|
||||
$config['date-format'] = $data['date_format'];
|
||||
$config['delimiter'] = $data['csv_delimiter'];
|
||||
$config['delimiter'] = $config['delimiter'] === 'tab' ? "\t" : $config['delimiter'];
|
||||
|
||||
Log::debug('Entered import account.', ['id' => $importId]);
|
||||
|
||||
|
||||
if (!is_null($account->id)) {
|
||||
Log::debug('Found account.', ['id' => $account->id, 'name' => $account->name]);
|
||||
$config['import-account'] = $account->id;
|
||||
} else {
|
||||
Log::error('Could not find anything for csv_import_account.', ['id' => $importId]);
|
||||
}
|
||||
// loop specifics.
|
||||
if (isset($data['specifics']) && is_array($data['specifics'])) {
|
||||
foreach ($data['specifics'] as $name => $enabled) {
|
||||
// verify their content.
|
||||
$className = sprintf('FireflyIII\Import\Specifics\%s', $name);
|
||||
if (class_exists($className)) {
|
||||
$config['specifics'][$name] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
|
||||
return true;
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*/
|
||||
public function setJob(ImportJob $job)
|
||||
{
|
||||
$this->job = $job;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the settings filled in by the user, if applicable.
|
||||
*
|
||||
* @param Request $request
|
||||
*
|
||||
*/
|
||||
public function storeSettings(Request $request)
|
||||
{
|
||||
$config = $this->job->configuration;
|
||||
$all = $request->all();
|
||||
if ($request->get('settings') == 'roles') {
|
||||
$count = $config['column-count'];
|
||||
$roleSet = 0; // how many roles have been defined
|
||||
$mapSet = 0; // how many columns must be mapped
|
||||
for ($i = 0; $i < $count; $i++) {
|
||||
$selectedRole = $all['role'][$i] ?? '_ignore';
|
||||
$doMapping = isset($all['map'][$i]) && $all['map'][$i] == '1' ? true : false;
|
||||
if ($selectedRole == '_ignore' && $doMapping === true) {
|
||||
$doMapping = false; // cannot map ignored columns.
|
||||
}
|
||||
if ($selectedRole != '_ignore') {
|
||||
$roleSet++;
|
||||
}
|
||||
if ($doMapping === true) {
|
||||
$mapSet++;
|
||||
}
|
||||
$config['column-roles'][$i] = $selectedRole;
|
||||
$config['column-do-mapping'][$i] = $doMapping;
|
||||
}
|
||||
if ($roleSet > 0) {
|
||||
$config['column-roles-complete'] = true;
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
}
|
||||
if ($mapSet === 0) {
|
||||
// skip setting of map:
|
||||
$config['column-mapping-complete'] = true;
|
||||
}
|
||||
}
|
||||
if ($request->get('settings') == 'map') {
|
||||
if (isset($all['mapping'])) {
|
||||
foreach ($all['mapping'] as $index => $data) {
|
||||
$config['column-mapping-config'][$index] = [];
|
||||
foreach ($data as $value => $mapId) {
|
||||
$mapId = intval($mapId);
|
||||
if ($mapId !== 0) {
|
||||
$config['column-mapping-config'][$index][$value] = intval($mapId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// set thing to be completed.
|
||||
$config['column-mapping-complete'] = true;
|
||||
$this->job->configuration = $config;
|
||||
$this->job->save();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,88 +0,0 @@
|
||||
<?php
|
||||
/**
|
||||
* SetupInterface.php
|
||||
* Copyright (C) 2016 thegrumpydictator@gmail.com
|
||||
*
|
||||
* This software may be modified and distributed under the terms of the
|
||||
* Creative Commons Attribution-ShareAlike 4.0 International License.
|
||||
*
|
||||
* See the LICENSE file for details.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace FireflyIII\Import\Setup;
|
||||
|
||||
use FireflyIII\Models\ImportJob;
|
||||
use Illuminate\Http\Request;
|
||||
use Symfony\Component\HttpFoundation\FileBag;
|
||||
|
||||
/**
|
||||
* Interface SetupInterface
|
||||
*
|
||||
* @package FireflyIII\Import\Setup
|
||||
*/
|
||||
interface SetupInterface
|
||||
{
|
||||
|
||||
/**
|
||||
* After uploading, and after setJob(), prepare anything that is
|
||||
* necessary for the configure() line.
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function configure(): bool;
|
||||
|
||||
/**
|
||||
* Returns any data necessary to do the configuration.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getConfigurationData(): array;
|
||||
|
||||
/**
|
||||
* This method returns the data required for the view that will let the user add settings to the import job.
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getDataForSettings(): array;
|
||||
|
||||
/**
|
||||
* This method returns the name of the view that will be shown to the user to further configure
|
||||
* the import job.
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getViewForSettings(): string;
|
||||
|
||||
/**
|
||||
* This method returns whether or not the user must configure this import
|
||||
* job further.
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function requireUserSettings(): bool;
|
||||
|
||||
/**
|
||||
* @param array $data
|
||||
*
|
||||
* @param FileBag $files
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function saveImportConfiguration(array $data, FileBag $files): bool;
|
||||
|
||||
/**
|
||||
* @param ImportJob $job
|
||||
*
|
||||
*/
|
||||
public function setJob(ImportJob $job);
|
||||
|
||||
/**
|
||||
* Store the settings filled in by the user, if applicable.
|
||||
*
|
||||
* @param Request $request
|
||||
*
|
||||
*/
|
||||
public function storeSettings(Request $request);
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
The import routine is as follows:
|
||||
|
||||
1. Upload and setup:
|
||||
|
||||
User uploads a file with entries. The Setup/SetupInterface gives the user
|
||||
the opportunity (in any number of steps) to do the necessary configuration.
|
||||
This could also be skipped of course. An ImportJob object is created with a
|
||||
basic and empty configuration.
|
||||
|
||||
Helper classes are as follows, greatly modelled to the CSV importer:
|
||||
|
||||
- The Mapper classes give back lists of Firefly objects. You can show them to the
|
||||
user in order to help you convert text values to their Firefly counterparts.
|
||||
For example, the user maps "Gcrsr" to category Groceries.
|
||||
- The Converter classes exist to help convert text values to their Firely counterparts.
|
||||
Feed "AB12ABCD897829" to the AssetAccountIban Converter and you should end up with a new
|
||||
or found asset account. The previously built mapping is used to narrow it down. Submit an empty
|
||||
mapping if this one is not relevant.
|
||||
|
||||
The mapping and possibly other configuration options are stored in a newly created
|
||||
ImportJob object, stored in the database. This import job holds a reference to the uploaded file
|
||||
(placed encrypted in /storage/uploads) and the status of the import.
|
||||
|
||||
2. Actual import
|
||||
|
||||
Using either the command line or the web interface the user can tell Firefly to start the import.
|
||||
|
||||
The ImporterInterface runs and creates an ImportEntry for each line, blob or whatever distinction it
|
||||
wants.
|
||||
|
||||
For each line, the ImporterInterface should run each field through the selected Converter in order
|
||||
to convert the text values to their Firefly counterparts. Again, this is modelled to the CSV importer
|
||||
and may need an update for MT940.
|
||||
|
||||
In any case, this newly minted set of ImportEntries (it cannot be saved or stored atm,
|
||||
this has to be done in one go) is then fed to the ImportValidator which will reject entries
|
||||
(almost never) and corrects fields if necessary.
|
||||
|
||||
|
||||
- Adds the date (today) if no date is present.
|
||||
- Determins the type of transaction (withdrawal, deposit, transfer).
|
||||
- Determins the types of accounts involved (asset, expense, revenue).
|
||||
- Determins the currency of the transaction.
|
||||
- Adds a default description if there isn't one present.
|
||||
|
||||
This set of corrected ImportEntries is then fed to the ImportStorage class which will generate
|
||||
TransactionJournals, Transactions and other related objects.
|
||||
Loading…
Reference in New Issue
Block a user